extlzham 0.0.1.PROTOTYPE

Sign up to get free protection for your applications and to get access to all the features.
Files changed (59) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.md +27 -0
  3. data/README.md +21 -0
  4. data/Rakefile +143 -0
  5. data/contrib/lzham/LICENSE +22 -0
  6. data/contrib/lzham/README.md +209 -0
  7. data/contrib/lzham/include/lzham.h +781 -0
  8. data/contrib/lzham/lzhamcomp/lzham_comp.h +38 -0
  9. data/contrib/lzham/lzhamcomp/lzham_lzbase.cpp +244 -0
  10. data/contrib/lzham/lzhamcomp/lzham_lzbase.h +45 -0
  11. data/contrib/lzham/lzhamcomp/lzham_lzcomp.cpp +608 -0
  12. data/contrib/lzham/lzhamcomp/lzham_lzcomp_internal.cpp +1966 -0
  13. data/contrib/lzham/lzhamcomp/lzham_lzcomp_internal.h +472 -0
  14. data/contrib/lzham/lzhamcomp/lzham_lzcomp_state.cpp +1413 -0
  15. data/contrib/lzham/lzhamcomp/lzham_match_accel.cpp +562 -0
  16. data/contrib/lzham/lzhamcomp/lzham_match_accel.h +146 -0
  17. data/contrib/lzham/lzhamcomp/lzham_null_threading.h +97 -0
  18. data/contrib/lzham/lzhamcomp/lzham_pthreads_threading.cpp +229 -0
  19. data/contrib/lzham/lzhamcomp/lzham_pthreads_threading.h +520 -0
  20. data/contrib/lzham/lzhamcomp/lzham_threading.h +12 -0
  21. data/contrib/lzham/lzhamcomp/lzham_win32_threading.cpp +220 -0
  22. data/contrib/lzham/lzhamcomp/lzham_win32_threading.h +368 -0
  23. data/contrib/lzham/lzhamdecomp/lzham_assert.cpp +66 -0
  24. data/contrib/lzham/lzhamdecomp/lzham_assert.h +40 -0
  25. data/contrib/lzham/lzhamdecomp/lzham_checksum.cpp +73 -0
  26. data/contrib/lzham/lzhamdecomp/lzham_checksum.h +13 -0
  27. data/contrib/lzham/lzhamdecomp/lzham_config.h +23 -0
  28. data/contrib/lzham/lzhamdecomp/lzham_core.h +264 -0
  29. data/contrib/lzham/lzhamdecomp/lzham_decomp.h +37 -0
  30. data/contrib/lzham/lzhamdecomp/lzham_helpers.h +54 -0
  31. data/contrib/lzham/lzhamdecomp/lzham_huffman_codes.cpp +262 -0
  32. data/contrib/lzham/lzhamdecomp/lzham_huffman_codes.h +14 -0
  33. data/contrib/lzham/lzhamdecomp/lzham_lzdecomp.cpp +1527 -0
  34. data/contrib/lzham/lzhamdecomp/lzham_lzdecompbase.cpp +131 -0
  35. data/contrib/lzham/lzhamdecomp/lzham_lzdecompbase.h +89 -0
  36. data/contrib/lzham/lzhamdecomp/lzham_math.h +142 -0
  37. data/contrib/lzham/lzhamdecomp/lzham_mem.cpp +284 -0
  38. data/contrib/lzham/lzhamdecomp/lzham_mem.h +112 -0
  39. data/contrib/lzham/lzhamdecomp/lzham_platform.cpp +157 -0
  40. data/contrib/lzham/lzhamdecomp/lzham_platform.h +284 -0
  41. data/contrib/lzham/lzhamdecomp/lzham_prefix_coding.cpp +351 -0
  42. data/contrib/lzham/lzhamdecomp/lzham_prefix_coding.h +146 -0
  43. data/contrib/lzham/lzhamdecomp/lzham_symbol_codec.cpp +1484 -0
  44. data/contrib/lzham/lzhamdecomp/lzham_symbol_codec.h +556 -0
  45. data/contrib/lzham/lzhamdecomp/lzham_timer.cpp +147 -0
  46. data/contrib/lzham/lzhamdecomp/lzham_timer.h +99 -0
  47. data/contrib/lzham/lzhamdecomp/lzham_traits.h +141 -0
  48. data/contrib/lzham/lzhamdecomp/lzham_types.h +97 -0
  49. data/contrib/lzham/lzhamdecomp/lzham_utils.h +58 -0
  50. data/contrib/lzham/lzhamdecomp/lzham_vector.cpp +75 -0
  51. data/contrib/lzham/lzhamdecomp/lzham_vector.h +588 -0
  52. data/contrib/lzham/lzhamlib/lzham_lib.cpp +179 -0
  53. data/examples/basic.rb +48 -0
  54. data/ext/extconf.rb +26 -0
  55. data/ext/extlzham.c +741 -0
  56. data/gemstub.rb +22 -0
  57. data/lib/extlzham/version.rb +5 -0
  58. data/lib/extlzham.rb +153 -0
  59. metadata +135 -0
@@ -0,0 +1,1413 @@
1
+ // File: lzham_lzcomp_state.cpp
2
+ // See Copyright Notice and license at the end of include/lzham.h
3
+ #include "lzham_core.h"
4
+ #include "lzham_lzcomp_internal.h"
5
+
6
+ namespace lzham
7
+ {
8
+ static uint get_huge_match_code_len(uint len)
9
+ {
10
+ LZHAM_ASSERT((len > CLZBase::cMaxMatchLen) && (len <= CLZBase::cMaxHugeMatchLen));
11
+ len -= (CLZBase::cMaxMatchLen + 1);
12
+
13
+ if (len < 256)
14
+ return 1 + 8;
15
+ else if (len < (256 + 1024))
16
+ return 2 + 10;
17
+ else if (len < (256 + 1024 + 4096))
18
+ return 3 + 12;
19
+ else
20
+ return 3 + 16;
21
+ }
22
+
23
+ static uint get_huge_match_code_bits(uint len)
24
+ {
25
+ LZHAM_ASSERT((len > CLZBase::cMaxMatchLen) && (len <= CLZBase::cMaxHugeMatchLen));
26
+ len -= (CLZBase::cMaxMatchLen + 1);
27
+
28
+ uint c;
29
+ if (len < 256)
30
+ c = len;
31
+ else if (len < (256 + 1024))
32
+ {
33
+ uint r = (len - 256);
34
+ LZHAM_ASSERT(r <= 1023);
35
+ c = r | (2 << 10);
36
+ }
37
+ else if (len < (256 + 1024 + 4096))
38
+ {
39
+ uint r = (len - (256 + 1024));
40
+ LZHAM_ASSERT(r <= 4095);
41
+ c = r | (6 << 12);
42
+ }
43
+ else
44
+ {
45
+ uint r = (len - (256 + 1024 + 4096));
46
+ LZHAM_ASSERT(r <= 65535);
47
+ c = r | (7 << 16);
48
+ }
49
+
50
+ return c;
51
+ }
52
+
53
+ uint lzcompressor::lzdecision::get_match_dist(const state& cur_state) const
54
+ {
55
+ if (!is_match())
56
+ return 0;
57
+ else if (is_rep())
58
+ {
59
+ int index = -m_dist - 1;
60
+ LZHAM_ASSERT(index < CLZBase::cMatchHistSize);
61
+ return cur_state.m_match_hist[index];
62
+ }
63
+ else
64
+ return m_dist;
65
+ }
66
+
67
+ lzcompressor::state::state()
68
+ {
69
+ m_cur_ofs = 0;
70
+ m_cur_state = 0;
71
+ m_block_start_dict_ofs = 0;
72
+
73
+ m_match_hist[0] = 1;
74
+ m_match_hist[1] = 1;
75
+ m_match_hist[2] = 1;
76
+ m_match_hist[3] = 1;
77
+ }
78
+
79
+ void lzcompressor::state::clear()
80
+ {
81
+ m_cur_ofs = 0;
82
+ m_cur_state = 0;
83
+ m_block_start_dict_ofs = 0;
84
+
85
+ for (uint i = 0; i < 2; i++)
86
+ {
87
+ m_rep_len_table[i].clear();
88
+ m_large_len_table[i].clear();
89
+ }
90
+ m_main_table.clear();
91
+ m_dist_lsb_table.clear();
92
+
93
+ m_lit_table.clear();
94
+ m_delta_lit_table.clear();
95
+
96
+ m_match_hist[0] = 1;
97
+ m_match_hist[1] = 1;
98
+ m_match_hist[2] = 1;
99
+ m_match_hist[3] = 1;
100
+ }
101
+
102
+ void lzcompressor::state::reset()
103
+ {
104
+ m_cur_ofs = 0;
105
+ m_cur_state = 0;
106
+ m_block_start_dict_ofs = 0;
107
+
108
+ for (uint i = 0; i < LZHAM_ARRAY_SIZE(m_is_match_model); i++)
109
+ m_is_match_model[i].clear();
110
+ for (uint i = 0; i < LZHAM_ARRAY_SIZE(m_is_rep_model); i++)
111
+ m_is_rep_model[i].clear();
112
+ for (uint i = 0; i < LZHAM_ARRAY_SIZE(m_is_rep0_model); i++)
113
+ m_is_rep0_model[i].clear();
114
+ for (uint i = 0; i < LZHAM_ARRAY_SIZE(m_is_rep0_single_byte_model); i++)
115
+ m_is_rep0_single_byte_model[i].clear();
116
+ for (uint i = 0; i < LZHAM_ARRAY_SIZE(m_is_rep1_model); i++)
117
+ m_is_rep1_model[i].clear();
118
+ for (uint i = 0; i < LZHAM_ARRAY_SIZE(m_is_rep2_model); i++)
119
+ m_is_rep2_model[i].clear();
120
+
121
+ for (uint i = 0; i < 2; i++)
122
+ {
123
+ m_rep_len_table[i].reset();
124
+ m_large_len_table[i].reset();
125
+ }
126
+ m_main_table.reset();
127
+ m_dist_lsb_table.reset();
128
+
129
+ m_lit_table.reset();
130
+ m_delta_lit_table.reset();
131
+
132
+ m_match_hist[0] = 1;
133
+ m_match_hist[1] = 1;
134
+ m_match_hist[2] = 1;
135
+ m_match_hist[3] = 1;
136
+ }
137
+
138
+ bool lzcompressor::state::init(CLZBase& lzbase, uint table_max_update_interval, uint table_update_interval_slow_rate)
139
+ {
140
+ m_cur_ofs = 0;
141
+ m_cur_state = 0;
142
+
143
+ if (!m_rep_len_table[0].init2(true, CLZBase::cNumHugeMatchCodes + (CLZBase::cMaxMatchLen - CLZBase::cMinMatchLen + 1), table_max_update_interval, table_update_interval_slow_rate, NULL))
144
+ return false;
145
+ if (!m_rep_len_table[1].assign(m_rep_len_table[0]))
146
+ return false;
147
+
148
+ if (!m_large_len_table[0].init2(true, CLZBase::cNumHugeMatchCodes + CLZBase::cLZXNumSecondaryLengths, table_max_update_interval, table_update_interval_slow_rate, NULL))
149
+ return false;
150
+ if (!m_large_len_table[1].assign(m_large_len_table[0]))
151
+ return false;
152
+
153
+ if (!m_main_table.init2(true, CLZBase::cLZXNumSpecialLengths + (lzbase.m_num_lzx_slots - CLZBase::cLZXLowestUsableMatchSlot) * 8, table_max_update_interval, table_update_interval_slow_rate, NULL))
154
+ return false;
155
+ if (!m_dist_lsb_table.init2(true, 16, table_max_update_interval, table_update_interval_slow_rate, NULL))
156
+ return false;
157
+
158
+ if (!m_lit_table.init2(true, 256, table_max_update_interval, table_update_interval_slow_rate, NULL))
159
+ return false;
160
+
161
+ if (!m_delta_lit_table.init2(true, 256, table_max_update_interval, table_update_interval_slow_rate, NULL))
162
+ return false;
163
+
164
+ m_match_hist[0] = 1;
165
+ m_match_hist[1] = 1;
166
+ m_match_hist[2] = 1;
167
+ m_match_hist[3] = 1;
168
+
169
+ return true;
170
+ }
171
+
172
+ void lzcompressor::state_base::partial_advance(const lzdecision& lzdec)
173
+ {
174
+ if (lzdec.m_len == 0)
175
+ {
176
+ if (m_cur_state < 4) m_cur_state = 0; else if (m_cur_state < 10) m_cur_state -= 3; else m_cur_state -= 6;
177
+ }
178
+ else
179
+ {
180
+ if (lzdec.m_dist < 0)
181
+ {
182
+ int match_hist_index = -lzdec.m_dist - 1;
183
+
184
+ if (!match_hist_index)
185
+ {
186
+ if (lzdec.m_len == 1)
187
+ {
188
+ m_cur_state = (m_cur_state < CLZBase::cNumLitStates) ? 9 : 11;
189
+ }
190
+ else
191
+ {
192
+ m_cur_state = (m_cur_state < CLZBase::cNumLitStates) ? 8 : 11;
193
+ }
194
+ }
195
+ else
196
+ {
197
+ if (match_hist_index == 1)
198
+ {
199
+ std::swap(m_match_hist[0], m_match_hist[1]);
200
+ }
201
+ else if (match_hist_index == 2)
202
+ {
203
+ int dist = m_match_hist[2];
204
+ m_match_hist[2] = m_match_hist[1];
205
+ m_match_hist[1] = m_match_hist[0];
206
+ m_match_hist[0] = dist;
207
+ }
208
+ else
209
+ {
210
+ LZHAM_ASSERT(match_hist_index == 3);
211
+
212
+ int dist = m_match_hist[3];
213
+ m_match_hist[3] = m_match_hist[2];
214
+ m_match_hist[2] = m_match_hist[1];
215
+ m_match_hist[1] = m_match_hist[0];
216
+ m_match_hist[0] = dist;
217
+ }
218
+
219
+ m_cur_state = (m_cur_state < CLZBase::cNumLitStates) ? 8 : 11;
220
+ }
221
+ }
222
+ else
223
+ {
224
+ // full
225
+ LZHAM_ASSUME(CLZBase::cMatchHistSize == 4);
226
+ m_match_hist[3] = m_match_hist[2];
227
+ m_match_hist[2] = m_match_hist[1];
228
+ m_match_hist[1] = m_match_hist[0];
229
+ m_match_hist[0] = lzdec.m_dist;
230
+
231
+ m_cur_state = (m_cur_state < CLZBase::cNumLitStates) ? CLZBase::cNumLitStates : CLZBase::cNumLitStates + 3;
232
+ }
233
+ }
234
+
235
+ m_cur_ofs = lzdec.m_pos + lzdec.get_len();
236
+ }
237
+
238
+ uint lzcompressor::state::get_pred_char(const search_accelerator& dict, int pos, int backward_ofs) const
239
+ {
240
+ LZHAM_ASSERT(pos >= (int)m_block_start_dict_ofs);
241
+ int limit = pos - m_block_start_dict_ofs;
242
+ if (backward_ofs > limit)
243
+ return 0;
244
+ return dict[pos - backward_ofs];
245
+ }
246
+
247
+ bit_cost_t lzcompressor::state::get_cost(CLZBase& lzbase, const search_accelerator& dict, const lzdecision& lzdec) const
248
+ {
249
+ //const uint lit_pred0 = get_pred_char(dict, lzdec.m_pos, 1);
250
+
251
+ uint is_match_model_index = LZHAM_IS_MATCH_MODEL_INDEX(m_cur_state);
252
+ LZHAM_ASSERT(is_match_model_index < LZHAM_ARRAY_SIZE(m_is_match_model));
253
+ bit_cost_t cost = m_is_match_model[is_match_model_index].get_cost(lzdec.is_match());
254
+
255
+ if (!lzdec.is_match())
256
+ {
257
+ const uint lit = dict[lzdec.m_pos];
258
+
259
+ if (m_cur_state < CLZBase::cNumLitStates)
260
+ {
261
+ // literal
262
+ cost += m_lit_table.get_cost(lit);
263
+ }
264
+ else
265
+ {
266
+ // delta literal
267
+ const uint rep_lit0 = dict[(lzdec.m_pos - m_match_hist[0]) & dict.m_max_dict_size_mask];
268
+
269
+ uint delta_lit = rep_lit0 ^ lit;
270
+
271
+ cost += m_delta_lit_table.get_cost(delta_lit);
272
+ }
273
+ }
274
+ else
275
+ {
276
+ // match
277
+ if (lzdec.m_dist < 0)
278
+ {
279
+ // rep match
280
+ cost += m_is_rep_model[m_cur_state].get_cost(1);
281
+
282
+ int match_hist_index = -lzdec.m_dist - 1;
283
+
284
+ if (!match_hist_index)
285
+ {
286
+ // rep0 match
287
+ cost += m_is_rep0_model[m_cur_state].get_cost(1);
288
+
289
+ if (lzdec.m_len == 1)
290
+ {
291
+ // single byte rep0
292
+ cost += m_is_rep0_single_byte_model[m_cur_state].get_cost(1);
293
+ }
294
+ else
295
+ {
296
+ // normal rep0
297
+ cost += m_is_rep0_single_byte_model[m_cur_state].get_cost(0);
298
+
299
+ if (lzdec.m_len > CLZBase::cMaxMatchLen)
300
+ {
301
+ cost += get_huge_match_code_len(lzdec.m_len) + m_rep_len_table[m_cur_state >= CLZBase::cNumLitStates].get_cost((CLZBase::cMaxMatchLen + 1) - CLZBase::cMinMatchLen);
302
+ }
303
+ else
304
+ {
305
+ cost += m_rep_len_table[m_cur_state >= CLZBase::cNumLitStates].get_cost(lzdec.m_len - CLZBase::cMinMatchLen);
306
+ }
307
+ }
308
+ }
309
+ else
310
+ {
311
+ if (lzdec.m_len > CLZBase::cMaxMatchLen)
312
+ {
313
+ cost += get_huge_match_code_len(lzdec.m_len) + m_rep_len_table[m_cur_state >= CLZBase::cNumLitStates].get_cost((CLZBase::cMaxMatchLen + 1) - CLZBase::cMinMatchLen);
314
+ }
315
+ else
316
+ {
317
+ cost += m_rep_len_table[m_cur_state >= CLZBase::cNumLitStates].get_cost(lzdec.m_len - CLZBase::cMinMatchLen);
318
+ }
319
+
320
+ // rep1-rep3 match
321
+ cost += m_is_rep0_model[m_cur_state].get_cost(0);
322
+
323
+ if (match_hist_index == 1)
324
+ {
325
+ // rep1
326
+ cost += m_is_rep1_model[m_cur_state].get_cost(1);
327
+ }
328
+ else
329
+ {
330
+ cost += m_is_rep1_model[m_cur_state].get_cost(0);
331
+
332
+ if (match_hist_index == 2)
333
+ {
334
+ // rep2
335
+ cost += m_is_rep2_model[m_cur_state].get_cost(1);
336
+ }
337
+ else
338
+ {
339
+ LZHAM_ASSERT(match_hist_index == 3);
340
+ // rep3
341
+ cost += m_is_rep2_model[m_cur_state].get_cost(0);
342
+ }
343
+ }
344
+ }
345
+ }
346
+ else
347
+ {
348
+ cost += m_is_rep_model[m_cur_state].get_cost(0);
349
+
350
+ LZHAM_ASSERT(lzdec.m_len >= CLZBase::cMinMatchLen);
351
+
352
+ // full match
353
+ uint match_slot, match_extra;
354
+ lzbase.compute_lzx_position_slot(lzdec.m_dist, match_slot, match_extra);
355
+
356
+ uint match_low_sym = 0;
357
+ if (lzdec.m_len >= 9)
358
+ {
359
+ match_low_sym = 7;
360
+ if (lzdec.m_len > CLZBase::cMaxMatchLen)
361
+ {
362
+ cost += get_huge_match_code_len(lzdec.m_len) + m_large_len_table[m_cur_state >= CLZBase::cNumLitStates].get_cost((CLZBase::cMaxMatchLen + 1) - 9);
363
+ }
364
+ else
365
+ {
366
+ cost += m_large_len_table[m_cur_state >= CLZBase::cNumLitStates].get_cost(lzdec.m_len - 9);
367
+ }
368
+ }
369
+ else
370
+ match_low_sym = lzdec.m_len - 2;
371
+
372
+ uint match_high_sym = 0;
373
+
374
+ LZHAM_ASSERT(match_slot >= CLZBase::cLZXLowestUsableMatchSlot && (match_slot < lzbase.m_num_lzx_slots));
375
+ match_high_sym = match_slot - CLZBase::cLZXLowestUsableMatchSlot;
376
+
377
+ uint main_sym = match_low_sym | (match_high_sym << 3);
378
+
379
+ cost += m_main_table.get_cost(CLZBase::cLZXNumSpecialLengths + main_sym);
380
+
381
+ uint num_extra_bits = lzbase.m_lzx_position_extra_bits[match_slot];
382
+ if (num_extra_bits < 3)
383
+ cost += convert_to_scaled_bitcost(num_extra_bits);
384
+ else
385
+ {
386
+ if (num_extra_bits > 4)
387
+ cost += convert_to_scaled_bitcost(num_extra_bits - 4);
388
+
389
+ cost += m_dist_lsb_table.get_cost(match_extra & 15);
390
+ }
391
+ }
392
+ }
393
+
394
+ return cost;
395
+ }
396
+
397
+ bit_cost_t lzcompressor::state::get_len2_match_cost(CLZBase& lzbase, uint dict_pos, uint len2_match_dist, uint is_match_model_index)
398
+ {
399
+ LZHAM_NOTE_UNUSED(dict_pos);
400
+
401
+ bit_cost_t cost = m_is_match_model[is_match_model_index].get_cost(1);
402
+
403
+ cost += m_is_rep_model[m_cur_state].get_cost(0);
404
+
405
+ // full match
406
+ uint match_slot, match_extra;
407
+ lzbase.compute_lzx_position_slot(len2_match_dist, match_slot, match_extra);
408
+
409
+ const uint match_len = 2;
410
+ uint match_low_sym = match_len - 2;
411
+
412
+ uint match_high_sym = 0;
413
+
414
+ LZHAM_ASSERT(match_slot >= CLZBase::cLZXLowestUsableMatchSlot && (match_slot < lzbase.m_num_lzx_slots));
415
+ match_high_sym = match_slot - CLZBase::cLZXLowestUsableMatchSlot;
416
+
417
+ uint main_sym = match_low_sym | (match_high_sym << 3);
418
+
419
+ cost += m_main_table.get_cost(CLZBase::cLZXNumSpecialLengths + main_sym);
420
+
421
+ uint num_extra_bits = lzbase.m_lzx_position_extra_bits[match_slot];
422
+ if (num_extra_bits < 3)
423
+ cost += convert_to_scaled_bitcost(num_extra_bits);
424
+ else
425
+ {
426
+ if (num_extra_bits > 4)
427
+ cost += convert_to_scaled_bitcost(num_extra_bits - 4);
428
+
429
+ cost += m_dist_lsb_table.get_cost(match_extra & 15);
430
+ }
431
+
432
+ return cost;
433
+ }
434
+
435
+ bit_cost_t lzcompressor::state::get_lit_cost(CLZBase& lzbase, const search_accelerator& dict, uint dict_pos, uint lit_pred0, uint is_match_model_index) const
436
+ {
437
+ LZHAM_NOTE_UNUSED(lzbase);
438
+ LZHAM_NOTE_UNUSED(lit_pred0);
439
+
440
+ bit_cost_t cost = m_is_match_model[is_match_model_index].get_cost(0);
441
+
442
+ const uint lit = dict[dict_pos];
443
+
444
+ if (m_cur_state < CLZBase::cNumLitStates)
445
+ {
446
+ // literal
447
+ cost += m_lit_table.get_cost(lit);
448
+ }
449
+ else
450
+ {
451
+ // delta literal
452
+ const uint rep_lit0 = dict[(dict_pos - m_match_hist[0]) & dict.m_max_dict_size_mask];
453
+
454
+ uint delta_lit = rep_lit0 ^ lit;
455
+
456
+ cost += m_delta_lit_table.get_cost(delta_lit);
457
+ }
458
+
459
+ return cost;
460
+ }
461
+
462
+ void lzcompressor::state::get_rep_match_costs(uint dict_pos, bit_cost_t *pBitcosts, uint match_hist_index, int min_len, int max_len, uint is_match_model_index) const
463
+ {
464
+ LZHAM_NOTE_UNUSED(dict_pos);
465
+ // match
466
+ const quasi_adaptive_huffman_data_model &rep_len_table = m_rep_len_table[m_cur_state >= CLZBase::cNumLitStates];
467
+
468
+ bit_cost_t base_cost = m_is_match_model[is_match_model_index].get_cost(1);
469
+
470
+ base_cost += m_is_rep_model[m_cur_state].get_cost(1);
471
+
472
+ if (!match_hist_index)
473
+ {
474
+ // rep0 match
475
+ base_cost += m_is_rep0_model[m_cur_state].get_cost(1);
476
+ }
477
+ else
478
+ {
479
+ // rep1-rep3 matches
480
+ base_cost += m_is_rep0_model[m_cur_state].get_cost(0);
481
+
482
+ if (match_hist_index == 1)
483
+ {
484
+ // rep1
485
+ base_cost += m_is_rep1_model[m_cur_state].get_cost(1);
486
+ }
487
+ else
488
+ {
489
+ base_cost += m_is_rep1_model[m_cur_state].get_cost(0);
490
+
491
+ if (match_hist_index == 2)
492
+ {
493
+ // rep2
494
+ base_cost += m_is_rep2_model[m_cur_state].get_cost(1);
495
+ }
496
+ else
497
+ {
498
+ // rep3
499
+ base_cost += m_is_rep2_model[m_cur_state].get_cost(0);
500
+ }
501
+ }
502
+ }
503
+
504
+ // rep match
505
+ if (!match_hist_index)
506
+ {
507
+ if (min_len == 1)
508
+ {
509
+ // single byte rep0
510
+ pBitcosts[1] = base_cost + m_is_rep0_single_byte_model[m_cur_state].get_cost(1);
511
+ min_len++;
512
+ }
513
+
514
+ bit_cost_t rep0_match_base_cost = base_cost + m_is_rep0_single_byte_model[m_cur_state].get_cost(0);
515
+ for (int match_len = min_len; match_len <= max_len; match_len++)
516
+ {
517
+ // normal rep0
518
+ if (match_len > CLZBase::cMaxMatchLen)
519
+ {
520
+ pBitcosts[match_len] = get_huge_match_code_len(match_len) + rep0_match_base_cost + rep_len_table.get_cost((CLZBase::cMaxMatchLen + 1) - CLZBase::cMinMatchLen);
521
+ }
522
+ else
523
+ {
524
+ pBitcosts[match_len] = rep0_match_base_cost + rep_len_table.get_cost(match_len - CLZBase::cMinMatchLen);
525
+ }
526
+ }
527
+ }
528
+ else
529
+ {
530
+ for (int match_len = min_len; match_len <= max_len; match_len++)
531
+ {
532
+ if (match_len > CLZBase::cMaxMatchLen)
533
+ {
534
+ pBitcosts[match_len] = get_huge_match_code_len(match_len) + base_cost + rep_len_table.get_cost((CLZBase::cMaxMatchLen + 1) - CLZBase::cMinMatchLen);
535
+ }
536
+ else
537
+ {
538
+ pBitcosts[match_len] = base_cost + rep_len_table.get_cost(match_len - CLZBase::cMinMatchLen);
539
+ }
540
+ }
541
+ }
542
+ }
543
+
544
+ void lzcompressor::state::get_full_match_costs(CLZBase& lzbase, uint dict_pos, bit_cost_t *pBitcosts, uint match_dist, int min_len, int max_len, uint is_match_model_index) const
545
+ {
546
+ LZHAM_NOTE_UNUSED(dict_pos);
547
+ LZHAM_ASSERT(min_len >= CLZBase::cMinMatchLen);
548
+
549
+ bit_cost_t cost = m_is_match_model[is_match_model_index].get_cost(1);
550
+
551
+ cost += m_is_rep_model[m_cur_state].get_cost(0);
552
+
553
+ uint match_slot, match_extra;
554
+ lzbase.compute_lzx_position_slot(match_dist, match_slot, match_extra);
555
+ LZHAM_ASSERT(match_slot >= CLZBase::cLZXLowestUsableMatchSlot && (match_slot < lzbase.m_num_lzx_slots));
556
+
557
+ uint num_extra_bits = lzbase.m_lzx_position_extra_bits[match_slot];
558
+
559
+ if (num_extra_bits < 3)
560
+ cost += convert_to_scaled_bitcost(num_extra_bits);
561
+ else
562
+ {
563
+ if (num_extra_bits > 4)
564
+ cost += convert_to_scaled_bitcost(num_extra_bits - 4);
565
+
566
+ cost += m_dist_lsb_table.get_cost(match_extra & 15);
567
+ }
568
+
569
+ uint match_high_sym = match_slot - CLZBase::cLZXLowestUsableMatchSlot;
570
+
571
+ const quasi_adaptive_huffman_data_model &large_len_table = m_large_len_table[m_cur_state >= CLZBase::cNumLitStates];
572
+
573
+ for (int match_len = min_len; match_len <= max_len; match_len++)
574
+ {
575
+ bit_cost_t len_cost = cost;
576
+
577
+ uint match_low_sym = 0;
578
+ if (match_len >= 9)
579
+ {
580
+ match_low_sym = 7;
581
+ if (match_len > CLZBase::cMaxMatchLen)
582
+ {
583
+ len_cost += get_huge_match_code_len(match_len) + large_len_table.get_cost((CLZBase::cMaxMatchLen + 1) - 9);
584
+ }
585
+ else
586
+ {
587
+ len_cost += large_len_table.get_cost(match_len - 9);
588
+ }
589
+ }
590
+ else
591
+ match_low_sym = match_len - 2;
592
+
593
+ uint main_sym = match_low_sym | (match_high_sym << 3);
594
+
595
+ pBitcosts[match_len] = len_cost + m_main_table.get_cost(CLZBase::cLZXNumSpecialLengths + main_sym);
596
+ }
597
+ }
598
+
599
+ bool lzcompressor::state::advance(CLZBase& lzbase, const search_accelerator& dict, const lzdecision& lzdec)
600
+ {
601
+ //const uint lit_pred0 = get_pred_char(dict, lzdec.m_pos, 1);
602
+
603
+ uint is_match_model_index = LZHAM_IS_MATCH_MODEL_INDEX(m_cur_state);
604
+ m_is_match_model[is_match_model_index].update(lzdec.is_match());
605
+
606
+ if (!lzdec.is_match())
607
+ {
608
+ const uint lit = dict[lzdec.m_pos];
609
+
610
+ if (m_cur_state < CLZBase::cNumLitStates)
611
+ {
612
+ // literal
613
+ if (!m_lit_table.update_sym(lit)) return false;
614
+ }
615
+ else
616
+ {
617
+ // delta literal
618
+ const uint rep_lit0 = dict[(lzdec.m_pos - m_match_hist[0]) & dict.m_max_dict_size_mask];
619
+
620
+ uint delta_lit = rep_lit0 ^ lit;
621
+
622
+ if (!m_delta_lit_table.update_sym(delta_lit)) return false;
623
+ }
624
+
625
+ if (m_cur_state < 4) m_cur_state = 0; else if (m_cur_state < 10) m_cur_state -= 3; else m_cur_state -= 6;
626
+ }
627
+ else
628
+ {
629
+ // match
630
+ if (lzdec.m_dist < 0)
631
+ {
632
+ // rep match
633
+ m_is_rep_model[m_cur_state].update(1);
634
+
635
+ int match_hist_index = -lzdec.m_dist - 1;
636
+
637
+ if (!match_hist_index)
638
+ {
639
+ // rep0 match
640
+ m_is_rep0_model[m_cur_state].update(1);
641
+
642
+ if (lzdec.m_len == 1)
643
+ {
644
+ // single byte rep0
645
+ m_is_rep0_single_byte_model[m_cur_state].update(1);
646
+
647
+ m_cur_state = (m_cur_state < CLZBase::cNumLitStates) ? 9 : 11;
648
+ }
649
+ else
650
+ {
651
+ // normal rep0
652
+ m_is_rep0_single_byte_model[m_cur_state].update(0);
653
+
654
+ if (lzdec.m_len > CLZBase::cMaxMatchLen)
655
+ {
656
+ if (!m_rep_len_table[m_cur_state >= CLZBase::cNumLitStates].update_sym((CLZBase::cMaxMatchLen + 1) - CLZBase::cMinMatchLen)) return false;
657
+ }
658
+ else
659
+ {
660
+ if (!m_rep_len_table[m_cur_state >= CLZBase::cNumLitStates].update_sym(lzdec.m_len - CLZBase::cMinMatchLen)) return false;
661
+ }
662
+
663
+ m_cur_state = (m_cur_state < CLZBase::cNumLitStates) ? 8 : 11;
664
+ }
665
+ }
666
+ else
667
+ {
668
+ // rep1-rep3 match
669
+ m_is_rep0_model[m_cur_state].update(0);
670
+
671
+ if (lzdec.m_len > CLZBase::cMaxMatchLen)
672
+ {
673
+ if (!m_rep_len_table[m_cur_state >= CLZBase::cNumLitStates].update_sym((CLZBase::cMaxMatchLen + 1) - CLZBase::cMinMatchLen)) return false;
674
+ }
675
+ else
676
+ {
677
+ if (!m_rep_len_table[m_cur_state >= CLZBase::cNumLitStates].update_sym(lzdec.m_len - CLZBase::cMinMatchLen)) return false;
678
+ }
679
+
680
+ if (match_hist_index == 1)
681
+ {
682
+ // rep1
683
+ m_is_rep1_model[m_cur_state].update(1);
684
+
685
+ std::swap(m_match_hist[0], m_match_hist[1]);
686
+ }
687
+ else
688
+ {
689
+ m_is_rep1_model[m_cur_state].update(0);
690
+
691
+ if (match_hist_index == 2)
692
+ {
693
+ // rep2
694
+ m_is_rep2_model[m_cur_state].update(1);
695
+
696
+ int dist = m_match_hist[2];
697
+ m_match_hist[2] = m_match_hist[1];
698
+ m_match_hist[1] = m_match_hist[0];
699
+ m_match_hist[0] = dist;
700
+ }
701
+ else
702
+ {
703
+ // rep3
704
+ m_is_rep2_model[m_cur_state].update(0);
705
+
706
+ int dist = m_match_hist[3];
707
+ m_match_hist[3] = m_match_hist[2];
708
+ m_match_hist[2] = m_match_hist[1];
709
+ m_match_hist[1] = m_match_hist[0];
710
+ m_match_hist[0] = dist;
711
+ }
712
+ }
713
+
714
+ m_cur_state = (m_cur_state < CLZBase::cNumLitStates) ? 8 : 11;
715
+ }
716
+ }
717
+ else
718
+ {
719
+ m_is_rep_model[m_cur_state].update(0);
720
+
721
+ LZHAM_ASSERT(lzdec.m_len >= CLZBase::cMinMatchLen);
722
+
723
+ // full match
724
+ uint match_slot, match_extra;
725
+ lzbase.compute_lzx_position_slot(lzdec.m_dist, match_slot, match_extra);
726
+
727
+ uint match_low_sym = 0;
728
+ int large_len_sym = -1;
729
+ if (lzdec.m_len >= 9)
730
+ {
731
+ match_low_sym = 7;
732
+
733
+ large_len_sym = lzdec.m_len - 9;
734
+ }
735
+ else
736
+ match_low_sym = lzdec.m_len - 2;
737
+
738
+ uint match_high_sym = 0;
739
+
740
+ LZHAM_ASSERT(match_slot >= CLZBase::cLZXLowestUsableMatchSlot && (match_slot < lzbase.m_num_lzx_slots));
741
+ match_high_sym = match_slot - CLZBase::cLZXLowestUsableMatchSlot;
742
+
743
+ uint main_sym = match_low_sym | (match_high_sym << 3);
744
+
745
+ if (!m_main_table.update_sym(CLZBase::cLZXNumSpecialLengths + main_sym)) return false;
746
+
747
+ if (large_len_sym >= 0)
748
+ {
749
+ if (lzdec.m_len > CLZBase::cMaxMatchLen)
750
+ {
751
+ if (!m_large_len_table[m_cur_state >= CLZBase::cNumLitStates].update_sym((CLZBase::cMaxMatchLen + 1) - 9)) return false;
752
+ }
753
+ else
754
+ {
755
+ if (!m_large_len_table[m_cur_state >= CLZBase::cNumLitStates].update_sym(large_len_sym)) return false;
756
+ }
757
+ }
758
+
759
+ uint num_extra_bits = lzbase.m_lzx_position_extra_bits[match_slot];
760
+ if (num_extra_bits >= 3)
761
+ {
762
+ if (!m_dist_lsb_table.update_sym(match_extra & 15)) return false;
763
+ }
764
+
765
+ update_match_hist(lzdec.m_dist);
766
+
767
+ m_cur_state = (m_cur_state < CLZBase::cNumLitStates) ? CLZBase::cNumLitStates : CLZBase::cNumLitStates + 3;
768
+ }
769
+ }
770
+
771
+ m_cur_ofs = lzdec.m_pos + lzdec.get_len();
772
+ return true;
773
+ }
774
+
775
+ bool lzcompressor::state::encode(symbol_codec& codec, CLZBase& lzbase, const search_accelerator& dict, const lzdecision& lzdec)
776
+ {
777
+ //const uint lit_pred0 = get_pred_char(dict, lzdec.m_pos, 1);
778
+
779
+ uint is_match_model_index = LZHAM_IS_MATCH_MODEL_INDEX(m_cur_state);
780
+ if (!codec.encode(lzdec.is_match(), m_is_match_model[is_match_model_index])) return false;
781
+
782
+ if (!lzdec.is_match())
783
+ {
784
+ const uint lit = dict[lzdec.m_pos];
785
+
786
+ #ifdef LZHAM_LZDEBUG
787
+ if (!codec.encode_bits(lit, 8)) return false;
788
+ #endif
789
+
790
+ if (m_cur_state < CLZBase::cNumLitStates)
791
+ {
792
+ // literal
793
+ if (!codec.encode(lit, m_lit_table)) return false;
794
+ }
795
+ else
796
+ {
797
+ // delta literal
798
+ const uint rep_lit0 = dict[(lzdec.m_pos - m_match_hist[0]) & dict.m_max_dict_size_mask];
799
+
800
+ uint delta_lit = rep_lit0 ^ lit;
801
+
802
+ #ifdef LZHAM_LZDEBUG
803
+ if (!codec.encode_bits(rep_lit0, 8)) return false;
804
+ #endif
805
+
806
+ if (!codec.encode(delta_lit, m_delta_lit_table)) return false;
807
+ }
808
+
809
+ if (m_cur_state < 4) m_cur_state = 0; else if (m_cur_state < 10) m_cur_state -= 3; else m_cur_state -= 6;
810
+ }
811
+ else
812
+ {
813
+ // match
814
+ if (lzdec.m_dist < 0)
815
+ {
816
+ // rep match
817
+ if (!codec.encode(1, m_is_rep_model[m_cur_state])) return false;
818
+
819
+ int match_hist_index = -lzdec.m_dist - 1;
820
+
821
+ if (!match_hist_index)
822
+ {
823
+ // rep0 match
824
+ if (!codec.encode(1, m_is_rep0_model[m_cur_state])) return false;
825
+
826
+ if (lzdec.m_len == 1)
827
+ {
828
+ // single byte rep0
829
+ if (!codec.encode(1, m_is_rep0_single_byte_model[m_cur_state])) return false;
830
+
831
+ m_cur_state = (m_cur_state < CLZBase::cNumLitStates) ? 9 : 11;
832
+ }
833
+ else
834
+ {
835
+ // normal rep0
836
+ if (!codec.encode(0, m_is_rep0_single_byte_model[m_cur_state])) return false;
837
+
838
+ if (lzdec.m_len > CLZBase::cMaxMatchLen)
839
+ {
840
+ if (!codec.encode((CLZBase::cMaxMatchLen + 1) - CLZBase::cMinMatchLen, m_rep_len_table[m_cur_state >= CLZBase::cNumLitStates])) return false;
841
+ if (!codec.encode_bits(get_huge_match_code_bits(lzdec.m_len), get_huge_match_code_len(lzdec.m_len))) return false;
842
+ }
843
+ else
844
+ {
845
+ if (!codec.encode(lzdec.m_len - CLZBase::cMinMatchLen, m_rep_len_table[m_cur_state >= CLZBase::cNumLitStates])) return false;
846
+ }
847
+
848
+ m_cur_state = (m_cur_state < CLZBase::cNumLitStates) ? 8 : 11;
849
+ }
850
+ }
851
+ else
852
+ {
853
+ // rep1-rep3 match
854
+ if (!codec.encode(0, m_is_rep0_model[m_cur_state])) return false;
855
+
856
+ if (lzdec.m_len > CLZBase::cMaxMatchLen)
857
+ {
858
+ if (!codec.encode((CLZBase::cMaxMatchLen + 1) - CLZBase::cMinMatchLen, m_rep_len_table[m_cur_state >= CLZBase::cNumLitStates])) return false;
859
+ if (!codec.encode_bits(get_huge_match_code_bits(lzdec.m_len), get_huge_match_code_len(lzdec.m_len))) return false;
860
+ }
861
+ else
862
+ {
863
+ if (!codec.encode(lzdec.m_len - CLZBase::cMinMatchLen, m_rep_len_table[m_cur_state >= CLZBase::cNumLitStates])) return false;
864
+ }
865
+
866
+ if (match_hist_index == 1)
867
+ {
868
+ // rep1
869
+ if (!codec.encode(1, m_is_rep1_model[m_cur_state])) return false;
870
+
871
+ std::swap(m_match_hist[0], m_match_hist[1]);
872
+ }
873
+ else
874
+ {
875
+ if (!codec.encode(0, m_is_rep1_model[m_cur_state])) return false;
876
+
877
+ if (match_hist_index == 2)
878
+ {
879
+ // rep2
880
+ if (!codec.encode(1, m_is_rep2_model[m_cur_state])) return false;
881
+
882
+ int dist = m_match_hist[2];
883
+ m_match_hist[2] = m_match_hist[1];
884
+ m_match_hist[1] = m_match_hist[0];
885
+ m_match_hist[0] = dist;
886
+ }
887
+ else
888
+ {
889
+ // rep3
890
+ if (!codec.encode(0, m_is_rep2_model[m_cur_state])) return false;
891
+
892
+ int dist = m_match_hist[3];
893
+ m_match_hist[3] = m_match_hist[2];
894
+ m_match_hist[2] = m_match_hist[1];
895
+ m_match_hist[1] = m_match_hist[0];
896
+ m_match_hist[0] = dist;
897
+ }
898
+ }
899
+
900
+ m_cur_state = (m_cur_state < CLZBase::cNumLitStates) ? 8 : 11;
901
+ }
902
+ }
903
+ else
904
+ {
905
+ if (!codec.encode(0, m_is_rep_model[m_cur_state])) return false;
906
+
907
+ LZHAM_ASSERT(lzdec.m_len >= CLZBase::cMinMatchLen);
908
+
909
+ // full match
910
+ uint match_slot, match_extra;
911
+ lzbase.compute_lzx_position_slot(lzdec.m_dist, match_slot, match_extra);
912
+
913
+ uint match_low_sym = 0;
914
+ int large_len_sym = -1;
915
+ if (lzdec.m_len >= 9)
916
+ {
917
+ match_low_sym = 7;
918
+
919
+ large_len_sym = lzdec.m_len - 9;
920
+ }
921
+ else
922
+ match_low_sym = lzdec.m_len - 2;
923
+
924
+ uint match_high_sym = 0;
925
+
926
+ LZHAM_ASSERT(match_slot >= CLZBase::cLZXLowestUsableMatchSlot && (match_slot < lzbase.m_num_lzx_slots));
927
+ match_high_sym = match_slot - CLZBase::cLZXLowestUsableMatchSlot;
928
+
929
+ uint main_sym = match_low_sym | (match_high_sym << 3);
930
+
931
+ if (!codec.encode(CLZBase::cLZXNumSpecialLengths + main_sym, m_main_table)) return false;
932
+
933
+ if (large_len_sym >= 0)
934
+ {
935
+ if (lzdec.m_len > CLZBase::cMaxMatchLen)
936
+ {
937
+ if (!codec.encode((CLZBase::cMaxMatchLen + 1) - 9, m_large_len_table[m_cur_state >= CLZBase::cNumLitStates])) return false;
938
+ if (!codec.encode_bits(get_huge_match_code_bits(lzdec.m_len), get_huge_match_code_len(lzdec.m_len))) return false;
939
+ }
940
+ else
941
+ {
942
+ if (!codec.encode(large_len_sym, m_large_len_table[m_cur_state >= CLZBase::cNumLitStates])) return false;
943
+ }
944
+ }
945
+
946
+ uint num_extra_bits = lzbase.m_lzx_position_extra_bits[match_slot];
947
+ if (num_extra_bits < 3)
948
+ {
949
+ if (!codec.encode_bits(match_extra, num_extra_bits)) return false;
950
+ }
951
+ else
952
+ {
953
+ if (num_extra_bits > 4)
954
+ {
955
+ if (!codec.encode_bits((match_extra >> 4), num_extra_bits - 4)) return false;
956
+ }
957
+
958
+ if (!codec.encode(match_extra & 15, m_dist_lsb_table)) return false;
959
+ }
960
+
961
+ update_match_hist(lzdec.m_dist);
962
+
963
+ m_cur_state = (m_cur_state < CLZBase::cNumLitStates) ? CLZBase::cNumLitStates : CLZBase::cNumLitStates + 3;
964
+ }
965
+
966
+ #ifdef LZHAM_LZDEBUG
967
+ if (!codec.encode_bits(m_match_hist[0], 29)) return false;
968
+ #endif
969
+ }
970
+
971
+ m_cur_ofs = lzdec.m_pos + lzdec.get_len();
972
+ return true;
973
+ }
974
+
975
+ void lzcompressor::state::print(symbol_codec& codec, CLZBase& lzbase, const search_accelerator& dict, const lzdecision& lzdec)
976
+ {
977
+ LZHAM_NOTE_UNUSED(codec), LZHAM_NOTE_UNUSED(lzbase), LZHAM_NOTE_UNUSED(dict);
978
+
979
+ const uint lit_pred0 = get_pred_char(dict, lzdec.m_pos, 1);
980
+
981
+ uint is_match_model_index = LZHAM_IS_MATCH_MODEL_INDEX(m_cur_state);
982
+
983
+ printf(" pos: %u, state: %u, match_pred: %u, is_match_model_index: %u, is_match: %u, cost: %f\n",
984
+ lzdec.m_pos,
985
+ m_cur_state,
986
+ lit_pred0, is_match_model_index, lzdec.is_match(), get_cost(lzbase, dict, lzdec) / (float)cBitCostScale);
987
+
988
+ if (!lzdec.is_match())
989
+ {
990
+ const uint lit = dict[lzdec.m_pos];
991
+
992
+ if (m_cur_state < CLZBase::cNumLitStates)
993
+ {
994
+ printf("---Regular lit: %u '%c'\n",
995
+ lit, ((lit >= 32) && (lit <= 127)) ? lit : '.');
996
+ }
997
+ else
998
+ {
999
+ // delta literal
1000
+ const uint rep_lit0 = dict[(lzdec.m_pos - m_match_hist[0]) & dict.m_max_dict_size_mask];
1001
+
1002
+ uint delta_lit = rep_lit0 ^ lit;
1003
+
1004
+ printf("***Delta lit: %u '%c', Mismatch: %u '%c', Delta: 0x%02X\n",
1005
+ lit, ((lit >= 32) && (lit <= 127)) ? lit : '.',
1006
+ rep_lit0, ((rep_lit0 >= 32) && (rep_lit0 <= 127)) ? rep_lit0 : '.',
1007
+ delta_lit);
1008
+ }
1009
+ }
1010
+ else
1011
+ {
1012
+ uint actual_match_len = dict.get_match_len(0, lzdec.get_match_dist(*this), CLZBase::cMaxMatchLen);
1013
+ LZHAM_ASSERT(actual_match_len >= lzdec.get_len());
1014
+
1015
+ // match
1016
+ if (lzdec.m_dist < 0)
1017
+ {
1018
+ int match_hist_index = -lzdec.m_dist - 1;
1019
+
1020
+ if (!match_hist_index)
1021
+ {
1022
+ if (lzdec.m_len == 1)
1023
+ {
1024
+ printf("!!!Rep 0 len1\n");
1025
+ }
1026
+ else
1027
+ {
1028
+ printf("!!!Rep 0 full len %u\n", lzdec.m_len);
1029
+ }
1030
+ }
1031
+ else
1032
+ {
1033
+ printf("!!!Rep %u full len %u\n", match_hist_index, lzdec.m_len);
1034
+ }
1035
+ }
1036
+ else
1037
+ {
1038
+ LZHAM_ASSERT(lzdec.m_len >= CLZBase::cMinMatchLen);
1039
+
1040
+ // full match
1041
+ uint match_slot, match_extra;
1042
+ lzbase.compute_lzx_position_slot(lzdec.m_dist, match_slot, match_extra);
1043
+
1044
+ uint match_low_sym = 0; LZHAM_NOTE_UNUSED(match_low_sym);
1045
+ int large_len_sym = -1; LZHAM_NOTE_UNUSED(large_len_sym);
1046
+ if (lzdec.m_len >= 9)
1047
+ {
1048
+ match_low_sym = 7;
1049
+
1050
+ large_len_sym = lzdec.m_len - 9;
1051
+ }
1052
+ else
1053
+ match_low_sym = lzdec.m_len - 2;
1054
+
1055
+ uint match_high_sym = 0; LZHAM_NOTE_UNUSED(match_high_sym);
1056
+
1057
+ LZHAM_ASSERT(match_slot >= CLZBase::cLZXLowestUsableMatchSlot && (match_slot < lzbase.m_num_lzx_slots));
1058
+ match_high_sym = match_slot - CLZBase::cLZXLowestUsableMatchSlot;
1059
+
1060
+ //uint main_sym = match_low_sym | (match_high_sym << 3);
1061
+
1062
+ uint num_extra_bits = lzbase.m_lzx_position_extra_bits[match_slot];
1063
+ printf("^^^Full match Len %u Dist %u, Slot %u, ExtraBits: %u", lzdec.m_len, lzdec.m_dist, match_slot, num_extra_bits);
1064
+
1065
+ if (num_extra_bits < 3)
1066
+ {
1067
+ }
1068
+ else
1069
+ {
1070
+ printf(" (Low 4 bits: %u vs. %u)", lzdec.m_dist & 15, match_extra & 15);
1071
+ }
1072
+ printf("\n");
1073
+ }
1074
+
1075
+ if (actual_match_len > lzdec.get_len())
1076
+ {
1077
+ printf(" TRUNCATED match, actual len is %u, shortened by %u\n", actual_match_len, actual_match_len - lzdec.get_len());
1078
+ }
1079
+ }
1080
+ }
1081
+
1082
+ bool lzcompressor::state::encode_eob(symbol_codec& codec, const search_accelerator& dict, uint dict_pos)
1083
+ {
1084
+ LZHAM_NOTE_UNUSED(dict);
1085
+ LZHAM_NOTE_UNUSED(dict_pos);
1086
+ #ifdef LZHAM_LZDEBUG
1087
+ if (!codec.encode_bits(CLZBase::cLZHAMDebugSyncMarkerValue, CLZBase::cLZHAMDebugSyncMarkerBits)) return false;
1088
+ if (!codec.encode_bits(1, 1)) return false;
1089
+ if (!codec.encode_bits(0, 17)) return false;
1090
+ if (!codec.encode_bits(m_cur_state, 4)) return false;
1091
+ #endif
1092
+
1093
+ //const uint match_pred = get_pred_char(dict, dict_pos, 1);
1094
+ uint is_match_model_index = LZHAM_IS_MATCH_MODEL_INDEX(m_cur_state);
1095
+ if (!codec.encode(1, m_is_match_model[is_match_model_index])) return false;
1096
+
1097
+ // full match
1098
+ if (!codec.encode(0, m_is_rep_model[m_cur_state])) return false;
1099
+
1100
+ return codec.encode(CLZBase::cLZXSpecialCodeEndOfBlockCode, m_main_table);
1101
+ }
1102
+
1103
+ bool lzcompressor::state::encode_reset_state_partial(symbol_codec& codec, const search_accelerator& dict, uint dict_pos)
1104
+ {
1105
+ LZHAM_NOTE_UNUSED(dict);
1106
+ LZHAM_NOTE_UNUSED(dict_pos);
1107
+ #ifdef LZHAM_LZDEBUG
1108
+ if (!codec.encode_bits(CLZBase::cLZHAMDebugSyncMarkerValue, CLZBase::cLZHAMDebugSyncMarkerBits)) return false;
1109
+ if (!codec.encode_bits(1, 1)) return false;
1110
+ if (!codec.encode_bits(0, 17)) return false;
1111
+ if (!codec.encode_bits(m_cur_state, 4)) return false;
1112
+ #endif
1113
+
1114
+ //const uint match_pred = get_pred_char(dict, dict_pos, 1);
1115
+ uint is_match_model_index = LZHAM_IS_MATCH_MODEL_INDEX(m_cur_state);
1116
+ if (!codec.encode(1, m_is_match_model[is_match_model_index])) return false;
1117
+
1118
+ // full match
1119
+ if (!codec.encode(0, m_is_rep_model[m_cur_state])) return false;
1120
+
1121
+ if (!codec.encode(CLZBase::cLZXSpecialCodePartialStateReset, m_main_table))
1122
+ return false;
1123
+
1124
+ reset_state_partial();
1125
+ return true;
1126
+ }
1127
+
1128
+ void lzcompressor::state::update_match_hist(uint match_dist)
1129
+ {
1130
+ LZHAM_ASSUME(CLZBase::cMatchHistSize == 4);
1131
+ m_match_hist[3] = m_match_hist[2];
1132
+ m_match_hist[2] = m_match_hist[1];
1133
+ m_match_hist[1] = m_match_hist[0];
1134
+ m_match_hist[0] = match_dist;
1135
+ }
1136
+
1137
+ int lzcompressor::state::find_match_dist(uint match_dist) const
1138
+ {
1139
+ for (uint match_hist_index = 0; match_hist_index < CLZBase::cMatchHistSize; match_hist_index++)
1140
+ if (match_dist == m_match_hist[match_hist_index])
1141
+ return match_hist_index;
1142
+
1143
+ return -1;
1144
+ }
1145
+
1146
+ void lzcompressor::state::reset_state_partial()
1147
+ {
1148
+ LZHAM_ASSUME(CLZBase::cMatchHistSize == 4);
1149
+ m_match_hist[0] = 1;
1150
+ m_match_hist[1] = 1;
1151
+ m_match_hist[2] = 1;
1152
+ m_match_hist[3] = 1;
1153
+ m_cur_state = 0;
1154
+ }
1155
+
1156
+ void lzcompressor::state::start_of_block(const search_accelerator& dict, uint cur_ofs, uint block_index)
1157
+ {
1158
+ LZHAM_NOTE_UNUSED(dict), LZHAM_NOTE_UNUSED(block_index);
1159
+
1160
+ reset_state_partial();
1161
+
1162
+ m_cur_ofs = cur_ofs;
1163
+ m_block_start_dict_ofs = cur_ofs;
1164
+ }
1165
+
1166
+ void lzcompressor::state::reset_update_rate()
1167
+ {
1168
+ m_lit_table.reset_update_rate();
1169
+ m_delta_lit_table.reset_update_rate();
1170
+
1171
+ m_main_table.reset_update_rate();
1172
+
1173
+ for (uint i = 0; i < LZHAM_ARRAY_SIZE(m_rep_len_table); i++)
1174
+ m_rep_len_table[i].reset_update_rate();
1175
+
1176
+ for (uint i = 0; i < LZHAM_ARRAY_SIZE(m_large_len_table); i++)
1177
+ m_large_len_table[i].reset_update_rate();
1178
+
1179
+ m_dist_lsb_table.reset_update_rate();
1180
+ }
1181
+
1182
+ void lzcompressor::coding_stats::clear()
1183
+ {
1184
+ m_total_bytes = 0;
1185
+ m_total_contexts = 0;
1186
+ m_total_match_bits_cost = 0;
1187
+ m_worst_match_bits_cost = 0;
1188
+ m_total_is_match0_bits_cost = 0;
1189
+ m_total_is_match1_bits_cost = 0;
1190
+ m_context_stats.clear();
1191
+
1192
+ m_total_nonmatches = 0;
1193
+ m_total_matches = 0;
1194
+ m_total_cost = 0.0f;
1195
+
1196
+ m_lit_stats.clear();
1197
+ m_delta_lit_stats.clear();
1198
+
1199
+ m_rep0_len1_stats.clear();
1200
+ for (uint i = 0; i < CLZBase::cMatchHistSize; i++)
1201
+ m_rep_stats[i].clear();
1202
+ m_rep0_len1_stats.clear();
1203
+ m_rep0_len2_plus_stats.clear();
1204
+
1205
+ for (uint i = 0; i <= CLZBase::cMaxMatchLen; i++)
1206
+ m_full_match_stats[i].clear();
1207
+
1208
+ m_total_far_len2_matches = 0;
1209
+ m_total_near_len2_matches = 0;
1210
+
1211
+ m_total_truncated_matches = 0;
1212
+ utils::zero_object(m_match_truncation_len_hist);
1213
+ utils::zero_object(m_match_truncation_hist);
1214
+ utils::zero_object(m_match_type_truncation_hist);
1215
+ utils::zero_object(m_match_type_was_not_truncated_hist);
1216
+
1217
+ m_total_update_rate_resets = 0;
1218
+
1219
+ m_max_len2_dist = 0;
1220
+ }
1221
+
1222
+ void lzcompressor::coding_stats::print()
1223
+ {
1224
+ if (!m_total_contexts)
1225
+ return;
1226
+
1227
+ printf("-----------\n");
1228
+ printf("Coding statistics:\n");
1229
+ printf("Total update rate resets: %u\n", m_total_update_rate_resets);
1230
+ printf("Total Bytes: %u, Total Contexts: %u, Total Cost: %f bits (%f bytes)\nContext ave cost: %f StdDev: %f Min: %f Max: %f\n", m_total_bytes, m_total_contexts, m_total_cost, m_total_cost / 8.0f, m_context_stats.get_average(), m_context_stats.get_std_dev(), m_context_stats.get_min_val(), m_context_stats.get_max_val());
1231
+ printf("Ave bytes per context: %f\n", m_total_bytes / (float)m_total_contexts);
1232
+
1233
+ printf("IsMatch:\n");
1234
+ printf(" Total: %u, Cost: %f (%f bytes), Ave. Cost: %f, Worst Cost: %f\n",
1235
+ m_total_contexts, m_total_match_bits_cost, m_total_match_bits_cost / 8.0f, m_total_match_bits_cost / math::maximum<uint>(1, m_total_contexts), m_worst_match_bits_cost);
1236
+
1237
+ printf(" IsMatch(0): %u, Cost: %f (%f bytes), Ave. Cost: %f\n",
1238
+ m_total_nonmatches, m_total_is_match0_bits_cost, m_total_is_match0_bits_cost / 8.0f, m_total_is_match0_bits_cost / math::maximum<uint>(1, m_total_nonmatches));
1239
+
1240
+ printf(" IsMatch(1): %u, Cost: %f (%f bytes), Ave. Cost: %f\n",
1241
+ m_total_matches, m_total_is_match1_bits_cost, m_total_is_match1_bits_cost / 8.0f, m_total_is_match1_bits_cost / math::maximum<uint>(1, m_total_matches));
1242
+
1243
+ printf("Literal stats:\n");
1244
+ printf(" Count: %u, Cost: %f (%f bytes), Ave: %f StdDev: %f Min: %f Max: %f\n", m_lit_stats.get_number_of_values32(), m_lit_stats.get_total(), m_lit_stats.get_total() / 8.0f, m_lit_stats.get_average(), m_lit_stats.get_std_dev(), m_lit_stats.get_min_val(), m_lit_stats.get_max_val());
1245
+
1246
+ printf("Delta literal stats:\n");
1247
+ printf(" Count: %u, Cost: %f (%f bytes), Ave: %f StdDev: %f Min: %f Max: %f\n", m_delta_lit_stats.get_number_of_values32(), m_delta_lit_stats.get_total(), m_delta_lit_stats.get_total() / 8.0f, m_delta_lit_stats.get_average(), m_delta_lit_stats.get_std_dev(), m_delta_lit_stats.get_min_val(), m_delta_lit_stats.get_max_val());
1248
+
1249
+ printf("Rep0 Len1 stats:\n");
1250
+ printf(" Count: %u, Cost: %f (%f bytes), Ave. Cost: %f StdDev: %f Min: %f Max: %f\n", m_rep0_len1_stats.get_number_of_values32(), m_rep0_len1_stats.get_total(), m_rep0_len1_stats.get_total() / 8.0f, m_rep0_len1_stats.get_average(), m_rep0_len1_stats.get_std_dev(), m_rep0_len1_stats.get_min_val(), m_rep0_len1_stats.get_max_val());
1251
+
1252
+ printf("Rep0 Len2+ stats:\n");
1253
+ printf(" Count: %u, Cost: %f (%f bytes), Ave. Cost: %f StdDev: %f Min: %f Max: %f\n", m_rep0_len2_plus_stats.get_number_of_values32(), m_rep0_len2_plus_stats.get_total(), m_rep0_len2_plus_stats.get_total() / 8.0f, m_rep0_len2_plus_stats.get_average(), m_rep0_len2_plus_stats.get_std_dev(), m_rep0_len2_plus_stats.get_min_val(), m_rep0_len2_plus_stats.get_max_val());
1254
+
1255
+ for (uint i = 0; i < CLZBase::cMatchHistSize; i++)
1256
+ {
1257
+ printf("Rep %u stats:\n", i);
1258
+ printf(" Count: %u, Cost: %f (%f bytes), Ave. Cost: %f StdDev: %f Min: %f Max: %f\n", m_rep_stats[i].get_number_of_values32(), m_rep_stats[i].get_total(), m_rep_stats[i].get_total() / 8.0f, m_rep_stats[i].get_average(), m_rep_stats[i].get_std_dev(), m_rep_stats[i].get_min_val(), m_rep_stats[i].get_max_val());
1259
+ }
1260
+
1261
+ for (uint i = CLZBase::cMinMatchLen; i <= CLZBase::cMaxMatchLen; i++)
1262
+ {
1263
+ printf("Match %u: Total: %u, Cost: %f (%f bytes), Ave: %f StdDev: %f Min: %f Max: %f\n", i,
1264
+ m_full_match_stats[i].get_number_of_values32(), m_full_match_stats[i].get_total(), m_full_match_stats[i].get_total() / 8.0f,
1265
+ m_full_match_stats[i].get_average(), m_full_match_stats[i].get_std_dev(), m_full_match_stats[i].get_min_val(), m_full_match_stats[i].get_max_val());
1266
+ }
1267
+
1268
+ printf("Total near len2 matches: %u, total far len2 matches: %u\n", m_total_near_len2_matches, m_total_far_len2_matches);
1269
+ printf("Total matches: %u, truncated matches: %u\n", m_total_matches, m_total_truncated_matches);
1270
+ printf("Max full match len2 distance: %u\n", m_max_len2_dist);
1271
+
1272
+ #if 0
1273
+ printf("Size of truncation histogram:\n");
1274
+ for (uint i = 0; i <= CLZBase::cMaxMatchLen; i++)
1275
+ {
1276
+ printf("%05u ", m_match_truncation_len_hist[i]);
1277
+ if ((i & 15) == 15) printf("\n");
1278
+ }
1279
+ printf("\n");
1280
+
1281
+ printf("Number of truncations per encoded match length histogram:\n");
1282
+ for (uint i = 0; i <= CLZBase::cMaxMatchLen; i++)
1283
+ {
1284
+ printf("%05u ", m_match_truncation_hist[i]);
1285
+ if ((i & 15) == 15) printf("\n");
1286
+ }
1287
+ printf("\n");
1288
+
1289
+ for (uint s = 0; s < CLZBase::cNumStates; s++)
1290
+ {
1291
+ printf("-- Match type truncation hist for state %u:\n", s);
1292
+ for (uint i = 0; i < LZHAM_ARRAY_SIZE(m_match_type_truncation_hist[s]); i++)
1293
+ {
1294
+ printf("%u truncated (%3.1f%%), %u not truncated\n", m_match_type_truncation_hist[s][i], 100.0f * (float)m_match_type_truncation_hist[s][i] / (m_match_type_truncation_hist[s][i] + m_match_type_was_not_truncated_hist[s][i]), m_match_type_was_not_truncated_hist[s][i]);
1295
+ }
1296
+ }
1297
+ #endif
1298
+ }
1299
+
1300
+ void lzcompressor::coding_stats::update(const lzdecision& lzdec, const state& cur_state, const search_accelerator& dict, bit_cost_t cost)
1301
+ {
1302
+ m_total_bytes += lzdec.get_len();
1303
+ m_total_contexts++;
1304
+
1305
+ float cost_in_bits = cost / (float)cBitCostScale;
1306
+ LZHAM_ASSERT(cost_in_bits > 0.0f);
1307
+ m_total_cost += cost_in_bits;
1308
+
1309
+ m_context_stats.update(cost_in_bits);
1310
+
1311
+ //uint match_pred = cur_state.get_pred_char(dict, lzdec.m_pos, 1);
1312
+ uint is_match_model_index = LZHAM_IS_MATCH_MODEL_INDEX(cur_state.m_cur_state);
1313
+
1314
+ if (lzdec.m_len == 0)
1315
+ {
1316
+ float match_bit_cost = cur_state.m_is_match_model[is_match_model_index].get_cost(0) / (float)cBitCostScale;
1317
+
1318
+ m_total_is_match0_bits_cost += match_bit_cost;
1319
+ m_total_match_bits_cost += match_bit_cost;
1320
+ m_worst_match_bits_cost = math::maximum<double>(m_worst_match_bits_cost, static_cast<double>(match_bit_cost));
1321
+ m_total_nonmatches++;
1322
+
1323
+ if (cur_state.m_cur_state < CLZBase::cNumLitStates)
1324
+ {
1325
+ m_lit_stats.update(cost_in_bits);
1326
+ }
1327
+ else
1328
+ {
1329
+ m_delta_lit_stats.update(cost_in_bits);
1330
+ }
1331
+ }
1332
+ else if (lzdec.m_len <= CLZBase::cMaxMatchLen)
1333
+ {
1334
+ const uint match_len = lzdec.get_len();
1335
+
1336
+ {
1337
+ uint match_dist = lzdec.get_match_dist(cur_state);
1338
+
1339
+ uint cur_lookahead_size = dict.get_lookahead_size();
1340
+
1341
+ uint actual_match_len = dict.get_match_len(0, match_dist, LZHAM_MIN(cur_lookahead_size, static_cast<uint>(CLZBase::cMaxMatchLen)));
1342
+ LZHAM_VERIFY(match_len <= actual_match_len);
1343
+
1344
+ m_total_truncated_matches += match_len < actual_match_len;
1345
+ m_match_truncation_len_hist[math::maximum<int>(0, actual_match_len - match_len)]++;
1346
+
1347
+ uint type_index = 4;
1348
+ if (!lzdec.is_full_match())
1349
+ {
1350
+ LZHAM_ASSUME(CLZBase::cMatchHistSize == 4);
1351
+ type_index = -lzdec.m_dist - 1;
1352
+ }
1353
+
1354
+ if (actual_match_len > match_len)
1355
+ {
1356
+ m_match_truncation_hist[match_len]++;
1357
+
1358
+ m_match_type_truncation_hist[cur_state.m_cur_state][type_index]++;
1359
+ }
1360
+ else
1361
+ {
1362
+ m_match_type_was_not_truncated_hist[cur_state.m_cur_state][type_index]++;
1363
+ }
1364
+ }
1365
+
1366
+ float match_bit_cost = cur_state.m_is_match_model[is_match_model_index].get_cost(1) / (float)cBitCostScale;
1367
+ m_total_is_match1_bits_cost += match_bit_cost;
1368
+ m_total_match_bits_cost += match_bit_cost;
1369
+ m_worst_match_bits_cost = math::maximum<double>(m_worst_match_bits_cost, static_cast<double>(match_bit_cost));
1370
+ m_total_matches++;
1371
+
1372
+ if (lzdec.m_dist < 0)
1373
+ {
1374
+ // rep match
1375
+ int match_hist_index = -lzdec.m_dist - 1;
1376
+ LZHAM_ASSERT(match_hist_index < CLZBase::cMatchHistSize);
1377
+
1378
+ m_rep_stats[match_hist_index].update(cost_in_bits);
1379
+
1380
+ if (!match_hist_index)
1381
+ {
1382
+ // rep0 match
1383
+ if (lzdec.m_len == 1)
1384
+ {
1385
+ m_rep0_len1_stats.update(cost_in_bits);
1386
+ }
1387
+ else
1388
+ {
1389
+ m_rep0_len2_plus_stats.update(cost_in_bits);
1390
+ }
1391
+ }
1392
+ }
1393
+ else
1394
+ {
1395
+ m_full_match_stats[math::minimum<int>(cMaxMatchLen, match_len)].update(cost_in_bits);
1396
+
1397
+ if (match_len == 2)
1398
+ {
1399
+ if (lzdec.m_dist <= 512)
1400
+ m_total_near_len2_matches++;
1401
+ else
1402
+ m_total_far_len2_matches++;
1403
+
1404
+ m_max_len2_dist = LZHAM_MAX((int)m_max_len2_dist, lzdec.m_dist);
1405
+ }
1406
+ }
1407
+ }
1408
+ else
1409
+ {
1410
+ // TODO: Handle huge matches.
1411
+ }
1412
+ }
1413
+ } // namespace lzham