isomorfeus-ferret 0.17.0 → 0.17.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/ext/isomorfeus_ferret_ext/bm_store.c +2 -0
- data/ext/isomorfeus_ferret_ext/frb_index.c +0 -35
- data/ext/isomorfeus_ferret_ext/frt_except.c +42 -0
- data/ext/isomorfeus_ferret_ext/frt_except.h +2 -0
- data/ext/isomorfeus_ferret_ext/frt_in_stream.c +492 -0
- data/ext/isomorfeus_ferret_ext/frt_in_stream.h +240 -0
- data/ext/isomorfeus_ferret_ext/frt_ind.c +0 -11
- data/ext/isomorfeus_ferret_ext/frt_ind.h +0 -1
- data/ext/isomorfeus_ferret_ext/frt_index.c +6 -1069
- data/ext/isomorfeus_ferret_ext/frt_index.h +1 -43
- data/ext/isomorfeus_ferret_ext/frt_lazy_doc.c +29 -0
- data/ext/isomorfeus_ferret_ext/frt_lazy_doc.h +19 -0
- data/ext/isomorfeus_ferret_ext/frt_lazy_doc_field.c +100 -0
- data/ext/isomorfeus_ferret_ext/frt_lazy_doc_field.h +33 -0
- data/ext/isomorfeus_ferret_ext/frt_out_stream.c +346 -0
- data/ext/isomorfeus_ferret_ext/frt_out_stream.h +198 -0
- data/ext/isomorfeus_ferret_ext/frt_ram_store.h +12 -0
- data/ext/isomorfeus_ferret_ext/frt_store.c +2 -513
- data/ext/isomorfeus_ferret_ext/frt_store.h +4 -443
- data/ext/isomorfeus_ferret_ext/frt_stream.h +18 -0
- data/ext/isomorfeus_ferret_ext/test_file_deleter.c +0 -1
- data/ext/isomorfeus_ferret_ext/test_index.c +0 -8
- data/ext/isomorfeus_ferret_ext/test_threading.c +1 -10
- data/lib/isomorfeus/ferret/index/index.rb +0 -11
- data/lib/isomorfeus/ferret/version.rb +1 -1
- metadata +22 -3
@@ -11,6 +11,7 @@
|
|
11
11
|
#include "frt_similarity.h"
|
12
12
|
#include "frt_bitvector.h"
|
13
13
|
#include "frt_priorityqueue.h"
|
14
|
+
#include "frt_lazy_doc.h"
|
14
15
|
|
15
16
|
typedef struct FrtIndexReader FrtIndexReader;
|
16
17
|
typedef struct FrtSegmentReader FrtSegmentReader;
|
@@ -495,48 +496,6 @@ extern int frt_tv_get_term_index(FrtTermVector *tv, const char *term);
|
|
495
496
|
extern int frt_tv_scan_to_term_index(FrtTermVector *tv, const char *term);
|
496
497
|
extern FrtTVTerm *frt_tv_get_tv_term(FrtTermVector *tv, const char *term);
|
497
498
|
|
498
|
-
/****************************************************************************
|
499
|
-
*
|
500
|
-
* FrtLazyDoc
|
501
|
-
*
|
502
|
-
****************************************************************************/
|
503
|
-
|
504
|
-
/* * * FrtLazyDocField * * */
|
505
|
-
typedef struct FrtLazyDocFieldData {
|
506
|
-
frt_off_t start;
|
507
|
-
int length;
|
508
|
-
rb_encoding *encoding;
|
509
|
-
FrtCompressionType compression; /* as stored */
|
510
|
-
char *text;
|
511
|
-
} FrtLazyDocFieldData;
|
512
|
-
|
513
|
-
typedef struct FrtLazyDoc FrtLazyDoc;
|
514
|
-
typedef struct FrtLazyDocField {
|
515
|
-
ID name;
|
516
|
-
FrtLazyDocFieldData *data;
|
517
|
-
FrtLazyDoc *doc;
|
518
|
-
int size; /* number of data elements */
|
519
|
-
int len; /* length of data elements concatenated */
|
520
|
-
FrtCompressionType compression; /* as configured */
|
521
|
-
bool decompressed;
|
522
|
-
bool loaded;
|
523
|
-
} FrtLazyDocField;
|
524
|
-
|
525
|
-
extern char *frt_lazy_df_get_data(FrtLazyDocField *self, int i);
|
526
|
-
extern void frt_lazy_df_get_bytes(FrtLazyDocField *self, char *buf, int start, int len);
|
527
|
-
|
528
|
-
/* * * FrtLazyDoc * * */
|
529
|
-
struct FrtLazyDoc {
|
530
|
-
FrtHash *field_dictionary;
|
531
|
-
int size;
|
532
|
-
FrtLazyDocField **fields;
|
533
|
-
FrtInStream *fields_in;
|
534
|
-
bool loaded;
|
535
|
-
};
|
536
|
-
|
537
|
-
extern void frt_lazy_doc_close(FrtLazyDoc *self);
|
538
|
-
extern FrtLazyDocField *frt_lazy_doc_get(FrtLazyDoc *self, ID field);
|
539
|
-
|
540
499
|
/****************************************************************************
|
541
500
|
*
|
542
501
|
* FrtFieldsReader
|
@@ -810,7 +769,6 @@ extern void frt_iw_close(FrtIndexWriter *iw);
|
|
810
769
|
extern void frt_iw_add_doc(FrtIndexWriter *iw, FrtDocument *doc);
|
811
770
|
extern int frt_iw_doc_count(FrtIndexWriter *iw);
|
812
771
|
extern void frt_iw_commit(FrtIndexWriter *iw);
|
813
|
-
extern void frt_iw_optimize(FrtIndexWriter *iw);
|
814
772
|
extern void frt_iw_add_readers(FrtIndexWriter *iw, FrtIndexReader **readers, const int r_cnt);
|
815
773
|
|
816
774
|
/****************************************************************************
|
@@ -0,0 +1,29 @@
|
|
1
|
+
#include "frt_lazy_doc.h"
|
2
|
+
|
3
|
+
FrtLazyDoc *frt_lazy_doc_new(int size, FrtInStream *fdt_in) {
|
4
|
+
FrtLazyDoc *self = FRT_ALLOC(FrtLazyDoc);
|
5
|
+
self->field_dictionary = frt_h_new_ptr((frt_free_ft)&frt_lazy_df_destroy);
|
6
|
+
self->size = size;
|
7
|
+
self->fields = FRT_ALLOC_AND_ZERO_N(FrtLazyDocField *, size);
|
8
|
+
self->fields_in = frt_is_clone(fdt_in);
|
9
|
+
self->loaded = false;
|
10
|
+
return self;
|
11
|
+
}
|
12
|
+
|
13
|
+
void frt_lazy_doc_close(FrtLazyDoc *self) {
|
14
|
+
frt_h_destroy(self->field_dictionary);
|
15
|
+
frt_is_close(self->fields_in);
|
16
|
+
free(self->fields);
|
17
|
+
free(self);
|
18
|
+
}
|
19
|
+
|
20
|
+
void frt_lazy_doc_add_field(FrtLazyDoc *self, FrtLazyDocField *lazy_df, int i) {
|
21
|
+
self->fields[i] = lazy_df;
|
22
|
+
|
23
|
+
frt_h_set(self->field_dictionary, (void *)lazy_df->name, lazy_df);
|
24
|
+
lazy_df->doc = self;
|
25
|
+
}
|
26
|
+
|
27
|
+
FrtLazyDocField *frt_lazy_doc_get(FrtLazyDoc *self, ID field) {
|
28
|
+
return (FrtLazyDocField *)frt_h_get(self->field_dictionary, (void *)field);
|
29
|
+
}
|
@@ -0,0 +1,19 @@
|
|
1
|
+
#ifndef FRT_LAZY_DOC_H
|
2
|
+
#define FRT_LAZY_DOC_H
|
3
|
+
|
4
|
+
#include "frt_lazy_doc_field.h"
|
5
|
+
|
6
|
+
typedef struct FrtLazyDoc {
|
7
|
+
FrtHash *field_dictionary;
|
8
|
+
int size;
|
9
|
+
FrtLazyDocField **fields;
|
10
|
+
FrtInStream *fields_in;
|
11
|
+
bool loaded;
|
12
|
+
} FrtLazyDoc;
|
13
|
+
|
14
|
+
extern FrtLazyDoc *frt_lazy_doc_new(int size, FrtInStream *fdt_in);
|
15
|
+
extern void frt_lazy_doc_add_field(FrtLazyDoc *self, FrtLazyDocField *lazy_df, int i);
|
16
|
+
extern void frt_lazy_doc_close(FrtLazyDoc *self);
|
17
|
+
extern FrtLazyDocField *frt_lazy_doc_get(FrtLazyDoc *self, ID field);
|
18
|
+
|
19
|
+
#endif
|
@@ -0,0 +1,100 @@
|
|
1
|
+
#include "frt_in_stream.h"
|
2
|
+
#include "frt_lazy_doc_field.h"
|
3
|
+
#include "frt_lazy_doc.h"
|
4
|
+
|
5
|
+
FrtLazyDocField *frt_lazy_df_new(ID name, const int size, FrtCompressionType compression) {
|
6
|
+
FrtLazyDocField *self = FRT_ALLOC(FrtLazyDocField);
|
7
|
+
self->name = name;
|
8
|
+
self->size = size;
|
9
|
+
self->data = FRT_ALLOC_AND_ZERO_N(FrtLazyDocFieldData, size);
|
10
|
+
self->compression = compression;
|
11
|
+
self->decompressed = false;
|
12
|
+
self->loaded = false;
|
13
|
+
return self;
|
14
|
+
}
|
15
|
+
|
16
|
+
void frt_lazy_df_destroy(FrtLazyDocField *self) {
|
17
|
+
int i;
|
18
|
+
for (i = self->size - 1; i >= 0; i--) {
|
19
|
+
if (self->data[i].text) {
|
20
|
+
free(self->data[i].text);
|
21
|
+
}
|
22
|
+
}
|
23
|
+
free(self->data);
|
24
|
+
free(self);
|
25
|
+
}
|
26
|
+
|
27
|
+
char *frt_lazy_df_get_data(FrtLazyDocField *self, int i) {
|
28
|
+
char *text = NULL;
|
29
|
+
if (i < self->size && i >= 0) {
|
30
|
+
text = self->data[i].text;
|
31
|
+
if (NULL == text) {
|
32
|
+
const int read_len = self->data[i].length + 1;
|
33
|
+
frt_is_seek(self->doc->fields_in, self->data[i].start);
|
34
|
+
if (self->data[i].compression != FRT_COMPRESSION_NONE) {
|
35
|
+
self->data[i].text = text = frt_is_read_compressed_bytes(self->doc->fields_in, read_len, &(self->data[i].length), self->data[i].compression);
|
36
|
+
} else {
|
37
|
+
self->data[i].text = text = FRT_ALLOC_N(char, read_len);
|
38
|
+
frt_is_read_bytes(self->doc->fields_in, (frt_uchar *)text, read_len);
|
39
|
+
text[read_len - 1] = '\0';
|
40
|
+
}
|
41
|
+
self->loaded = true;
|
42
|
+
}
|
43
|
+
}
|
44
|
+
|
45
|
+
return text;
|
46
|
+
}
|
47
|
+
|
48
|
+
void frt_lazy_df_get_bytes(FrtLazyDocField *self, char *buf, int start, int len) {
|
49
|
+
if (self->compression != FRT_COMPRESSION_NONE && !self->decompressed) {
|
50
|
+
int i;
|
51
|
+
self->len = 0;
|
52
|
+
for (i = self->size-1; i >= 0; i--) {
|
53
|
+
(void)frt_lazy_df_get_data(self, i);
|
54
|
+
self->len += self->data[i].length + 1;
|
55
|
+
}
|
56
|
+
self->len--; /* each field separated by ' ' but no need to add to end */
|
57
|
+
self->decompressed = true;
|
58
|
+
}
|
59
|
+
if (start < 0 || start >= self->len) {
|
60
|
+
FRT_RAISE(FRT_IO_ERROR, "start out of range in LazyDocField#get_bytes. %d "
|
61
|
+
"is not between 0 and %d", start, self->len);
|
62
|
+
}
|
63
|
+
if (len <= 0) {
|
64
|
+
FRT_RAISE(FRT_IO_ERROR, "len = %d, but should be greater than 0", len);
|
65
|
+
}
|
66
|
+
if (start + len > self->len) {
|
67
|
+
FRT_RAISE(FRT_IO_ERROR, "Tried to read past end of field. Field is only %d "
|
68
|
+
"bytes long but tried to read to %d", self->len, start + len);
|
69
|
+
}
|
70
|
+
if (self->compression != FRT_COMPRESSION_NONE) {
|
71
|
+
int cur_start = 0, buf_start = 0, cur_end, i, copy_start, copy_len;
|
72
|
+
for (i = 0; i < self->size; i++) {
|
73
|
+
cur_end = cur_start + self->data[i].length;
|
74
|
+
if (start < cur_end) {
|
75
|
+
copy_start = start > cur_start ? start - cur_start : 0;
|
76
|
+
copy_len = cur_end - cur_start - copy_start;
|
77
|
+
if (copy_len >= len) {
|
78
|
+
copy_len = len;
|
79
|
+
len = 0;
|
80
|
+
}
|
81
|
+
else {
|
82
|
+
len -= copy_len;
|
83
|
+
}
|
84
|
+
memcpy(buf + buf_start,
|
85
|
+
self->data[i].text + copy_start,
|
86
|
+
copy_len);
|
87
|
+
buf_start += copy_len;
|
88
|
+
if (len > 0) {
|
89
|
+
buf[buf_start++] = ' ';
|
90
|
+
len--;
|
91
|
+
}
|
92
|
+
if (len == 0) break;
|
93
|
+
}
|
94
|
+
cur_start = cur_end + 1;
|
95
|
+
}
|
96
|
+
} else {
|
97
|
+
frt_is_seek(self->doc->fields_in, self->data[0].start + start);
|
98
|
+
frt_is_read_bytes(self->doc->fields_in, (frt_uchar *)buf, len);
|
99
|
+
}
|
100
|
+
}
|
@@ -0,0 +1,33 @@
|
|
1
|
+
#ifndef FRT_LAZY_DOC_FIELD_H
|
2
|
+
#define FRT_LAZY_DOC_FIELD_H
|
3
|
+
|
4
|
+
#include <ruby/encoding.h>
|
5
|
+
#include "frt_hash.h"
|
6
|
+
#include "frt_in_stream.h"
|
7
|
+
|
8
|
+
typedef struct FrtLazyDocFieldData {
|
9
|
+
frt_off_t start;
|
10
|
+
int length;
|
11
|
+
rb_encoding *encoding;
|
12
|
+
FrtCompressionType compression; /* as stored */
|
13
|
+
char *text;
|
14
|
+
} FrtLazyDocFieldData;
|
15
|
+
|
16
|
+
typedef struct FrtLazyDoc FrtLazyDoc;
|
17
|
+
typedef struct FrtLazyDocField {
|
18
|
+
ID name;
|
19
|
+
FrtLazyDocFieldData *data;
|
20
|
+
FrtLazyDoc *doc;
|
21
|
+
int size; /* number of data elements */
|
22
|
+
int len; /* length of data elements concatenated */
|
23
|
+
FrtCompressionType compression; /* as configured */
|
24
|
+
bool decompressed;
|
25
|
+
bool loaded;
|
26
|
+
} FrtLazyDocField;
|
27
|
+
|
28
|
+
extern FrtLazyDocField *frt_lazy_df_new(ID name, const int size, FrtCompressionType compression);
|
29
|
+
extern char *frt_lazy_df_get_data(FrtLazyDocField *self, int i);
|
30
|
+
extern void frt_lazy_df_get_bytes(FrtLazyDocField *self, char *buf, int start, int len);
|
31
|
+
extern void frt_lazy_df_destroy(FrtLazyDocField *self);
|
32
|
+
|
33
|
+
#endif
|
@@ -0,0 +1,346 @@
|
|
1
|
+
#include "brotli_decode.h"
|
2
|
+
#include "brotli_encode.h"
|
3
|
+
#include "bzlib.h"
|
4
|
+
#include "lz4frame.h"
|
5
|
+
#include "frt_except.h"
|
6
|
+
#include "frt_stream.h"
|
7
|
+
#include "frt_out_stream.h"
|
8
|
+
|
9
|
+
#define FRT_BROTLI_COMPRESSION_LEVEL 4
|
10
|
+
#define FRT_BZIP_COMPRESSION_LEVEL 9
|
11
|
+
|
12
|
+
/**
|
13
|
+
* Create a newly allocated and initialized OutStream object
|
14
|
+
*
|
15
|
+
* @return a newly allocated and initialized OutStream object
|
16
|
+
*/
|
17
|
+
FrtOutStream *frt_os_new(void) {
|
18
|
+
FrtOutStream *os = FRT_ALLOC(FrtOutStream);
|
19
|
+
os->buf.start = 0;
|
20
|
+
os->buf.pos = 0;
|
21
|
+
os->buf.len = 0;
|
22
|
+
return os;
|
23
|
+
}
|
24
|
+
|
25
|
+
/**
|
26
|
+
* Flush the countents of the FrtOutStream's buffers
|
27
|
+
*
|
28
|
+
* @param the OutStream to flush
|
29
|
+
*/
|
30
|
+
void frt_os_flush(FrtOutStream *os)
|
31
|
+
{
|
32
|
+
os->m->flush_i(os, os->buf.buf, os->buf.pos);
|
33
|
+
os->buf.start += os->buf.pos;
|
34
|
+
os->buf.pos = 0;
|
35
|
+
}
|
36
|
+
|
37
|
+
void frt_os_close(FrtOutStream *os)
|
38
|
+
{
|
39
|
+
frt_os_flush(os);
|
40
|
+
os->m->close_i(os);
|
41
|
+
free(os);
|
42
|
+
}
|
43
|
+
|
44
|
+
off_t frt_os_pos(FrtOutStream *os)
|
45
|
+
{
|
46
|
+
return os->buf.start + os->buf.pos;
|
47
|
+
}
|
48
|
+
|
49
|
+
void frt_os_seek(FrtOutStream *os, frt_off_t new_pos)
|
50
|
+
{
|
51
|
+
frt_os_flush(os);
|
52
|
+
os->buf.start = new_pos;
|
53
|
+
os->m->seek_i(os, new_pos);
|
54
|
+
}
|
55
|
+
|
56
|
+
/**
|
57
|
+
* Unsafe alternative to os_write_byte. Only use this method if you know there
|
58
|
+
* is no chance of buffer overflow.
|
59
|
+
*/
|
60
|
+
#define write_byte(os, b) os->buf.buf[os->buf.pos++] = (frt_uchar)b
|
61
|
+
|
62
|
+
/**
|
63
|
+
* Write a single byte +b+ to the OutStream +os+
|
64
|
+
*
|
65
|
+
* @param os the OutStream to write to
|
66
|
+
* @param b the byte to write
|
67
|
+
* @raise FRT_IO_ERROR if there is an IO error writing to the filesystem
|
68
|
+
*/
|
69
|
+
void frt_os_write_byte(FrtOutStream *os, frt_uchar b)
|
70
|
+
{
|
71
|
+
if (os->buf.pos >= (FRT_BUFFER_SIZE - 1)) {
|
72
|
+
frt_os_flush(os);
|
73
|
+
}
|
74
|
+
write_byte(os, b);
|
75
|
+
}
|
76
|
+
|
77
|
+
void frt_os_write_bytes(FrtOutStream *os, const frt_uchar *buf, int len)
|
78
|
+
{
|
79
|
+
if (len < (FRT_BUFFER_SIZE - os->buf.pos)) {
|
80
|
+
memcpy(os->buf.buf + os->buf.pos, buf, len);
|
81
|
+
os->buf.pos += len;
|
82
|
+
}
|
83
|
+
else {
|
84
|
+
frt_os_flush(os);
|
85
|
+
int pos = 0;
|
86
|
+
int size;
|
87
|
+
while (pos < len) {
|
88
|
+
if (len - pos < FRT_BUFFER_SIZE) {
|
89
|
+
size = len - pos;
|
90
|
+
}
|
91
|
+
else {
|
92
|
+
size = FRT_BUFFER_SIZE;
|
93
|
+
}
|
94
|
+
os->m->flush_i(os, buf + pos, size);
|
95
|
+
pos += size;
|
96
|
+
os->buf.start += size;
|
97
|
+
}
|
98
|
+
}
|
99
|
+
}
|
100
|
+
|
101
|
+
void frt_os_write_i32(FrtOutStream *os, frt_i32 num)
|
102
|
+
{
|
103
|
+
frt_os_write_byte(os, (frt_uchar)((num >> 24) & 0xFF));
|
104
|
+
frt_os_write_byte(os, (frt_uchar)((num >> 16) & 0xFF));
|
105
|
+
frt_os_write_byte(os, (frt_uchar)((num >> 8) & 0xFF));
|
106
|
+
frt_os_write_byte(os, (frt_uchar)(num & 0xFF));
|
107
|
+
}
|
108
|
+
|
109
|
+
void frt_os_write_i64(FrtOutStream *os, frt_i64 num)
|
110
|
+
{
|
111
|
+
frt_os_write_byte(os, (frt_uchar)((num >> 56) & 0xFF));
|
112
|
+
frt_os_write_byte(os, (frt_uchar)((num >> 48) & 0xFF));
|
113
|
+
frt_os_write_byte(os, (frt_uchar)((num >> 40) & 0xFF));
|
114
|
+
frt_os_write_byte(os, (frt_uchar)((num >> 32) & 0xFF));
|
115
|
+
frt_os_write_byte(os, (frt_uchar)((num >> 24) & 0xFF));
|
116
|
+
frt_os_write_byte(os, (frt_uchar)((num >> 16) & 0xFF));
|
117
|
+
frt_os_write_byte(os, (frt_uchar)((num >> 8) & 0xFF));
|
118
|
+
frt_os_write_byte(os, (frt_uchar)(num & 0xFF));
|
119
|
+
}
|
120
|
+
|
121
|
+
void frt_os_write_u32(FrtOutStream *os, frt_u32 num)
|
122
|
+
{
|
123
|
+
frt_os_write_byte(os, (frt_uchar)((num >> 24) & 0xFF));
|
124
|
+
frt_os_write_byte(os, (frt_uchar)((num >> 16) & 0xFF));
|
125
|
+
frt_os_write_byte(os, (frt_uchar)((num >> 8) & 0xFF));
|
126
|
+
frt_os_write_byte(os, (frt_uchar)(num & 0xFF));
|
127
|
+
}
|
128
|
+
|
129
|
+
void frt_os_write_u64(FrtOutStream *os, frt_u64 num)
|
130
|
+
{
|
131
|
+
frt_os_write_byte(os, (frt_uchar)((num >> 56) & 0xFF));
|
132
|
+
frt_os_write_byte(os, (frt_uchar)((num >> 48) & 0xFF));
|
133
|
+
frt_os_write_byte(os, (frt_uchar)((num >> 40) & 0xFF));
|
134
|
+
frt_os_write_byte(os, (frt_uchar)((num >> 32) & 0xFF));
|
135
|
+
frt_os_write_byte(os, (frt_uchar)((num >> 24) & 0xFF));
|
136
|
+
frt_os_write_byte(os, (frt_uchar)((num >> 16) & 0xFF));
|
137
|
+
frt_os_write_byte(os, (frt_uchar)((num >> 8) & 0xFF));
|
138
|
+
frt_os_write_byte(os, (frt_uchar)(num & 0xFF));
|
139
|
+
}
|
140
|
+
|
141
|
+
/* optimized to use an unchecked write if there is space */
|
142
|
+
void frt_os_write_vint(FrtOutStream *os, register unsigned int num)
|
143
|
+
{
|
144
|
+
if (os->buf.pos > VINT_END) {
|
145
|
+
while (num > 127) {
|
146
|
+
frt_os_write_byte(os, (frt_uchar)((num & 0x7f) | 0x80));
|
147
|
+
num >>= 7;
|
148
|
+
}
|
149
|
+
frt_os_write_byte(os, (frt_uchar)(num));
|
150
|
+
}
|
151
|
+
else {
|
152
|
+
while (num > 127) {
|
153
|
+
write_byte(os, (frt_uchar)((num & 0x7f) | 0x80));
|
154
|
+
num >>= 7;
|
155
|
+
}
|
156
|
+
write_byte(os, (frt_uchar)(num));
|
157
|
+
}
|
158
|
+
}
|
159
|
+
|
160
|
+
/* optimized to use an unchecked write if there is space */
|
161
|
+
void frt_os_write_voff_t(FrtOutStream *os, register frt_off_t num)
|
162
|
+
{
|
163
|
+
if (os->buf.pos > VINT_END) {
|
164
|
+
while (num > 127) {
|
165
|
+
frt_os_write_byte(os, (frt_uchar)((num & 0x7f) | 0x80));
|
166
|
+
num >>= 7;
|
167
|
+
}
|
168
|
+
frt_os_write_byte(os, (frt_uchar)num);
|
169
|
+
}
|
170
|
+
else {
|
171
|
+
while (num > 127) {
|
172
|
+
write_byte(os, (frt_uchar)((num & 0x7f) | 0x80));
|
173
|
+
num >>= 7;
|
174
|
+
}
|
175
|
+
write_byte(os, (frt_uchar)num);
|
176
|
+
}
|
177
|
+
}
|
178
|
+
|
179
|
+
/* optimized to use an unchecked write if there is space */
|
180
|
+
void frt_os_write_vll(FrtOutStream *os, register frt_u64 num)
|
181
|
+
{
|
182
|
+
if (os->buf.pos > VINT_END) {
|
183
|
+
while (num > 127) {
|
184
|
+
frt_os_write_byte(os, (frt_uchar)((num & 0x7f) | 0x80));
|
185
|
+
num >>= 7;
|
186
|
+
}
|
187
|
+
frt_os_write_byte(os, (frt_uchar)num);
|
188
|
+
}
|
189
|
+
else {
|
190
|
+
while (num > 127) {
|
191
|
+
write_byte(os, (frt_uchar)((num & 0x7f) | 0x80));
|
192
|
+
num >>= 7;
|
193
|
+
}
|
194
|
+
write_byte(os, (frt_uchar)num);
|
195
|
+
}
|
196
|
+
}
|
197
|
+
|
198
|
+
void frt_os_write_string_len(FrtOutStream *os, const char *str, int len)
|
199
|
+
{
|
200
|
+
frt_os_write_vint(os, len);
|
201
|
+
frt_os_write_bytes(os, (frt_uchar *)str, len);
|
202
|
+
}
|
203
|
+
|
204
|
+
void frt_os_write_string(FrtOutStream *os, const char *str)
|
205
|
+
{
|
206
|
+
frt_os_write_string_len(os, str, (int)strlen(str));
|
207
|
+
}
|
208
|
+
|
209
|
+
static int frt_os_write_brotli_compressed_bytes(FrtOutStream* out_stream, frt_uchar *data, int length) {
|
210
|
+
size_t compressed_length = 0;
|
211
|
+
const frt_uchar *next_in = data;
|
212
|
+
size_t available_in = length;
|
213
|
+
size_t available_out;
|
214
|
+
frt_uchar compression_buffer[FRT_COMPRESSION_BUFFER_SIZE];
|
215
|
+
frt_uchar *next_out;
|
216
|
+
BrotliEncoderState *b_state = BrotliEncoderCreateInstance(NULL, NULL, NULL);
|
217
|
+
if (!b_state) { frt_comp_raise(); return -1; }
|
218
|
+
|
219
|
+
BrotliEncoderSetParameter(b_state, BROTLI_PARAM_QUALITY, FRT_BROTLI_COMPRESSION_LEVEL);
|
220
|
+
|
221
|
+
do {
|
222
|
+
available_out = FRT_COMPRESSION_BUFFER_SIZE;
|
223
|
+
next_out = compression_buffer;
|
224
|
+
if (!BrotliEncoderCompressStream(b_state, BROTLI_OPERATION_FINISH,
|
225
|
+
&available_in, &next_in,
|
226
|
+
&available_out, &next_out, &compressed_length)) {
|
227
|
+
BrotliEncoderDestroyInstance(b_state);
|
228
|
+
frt_comp_raise();
|
229
|
+
return -1;
|
230
|
+
}
|
231
|
+
frt_os_write_bytes(out_stream, compression_buffer, FRT_COMPRESSION_BUFFER_SIZE - available_out);
|
232
|
+
} while (!BrotliEncoderIsFinished(b_state));
|
233
|
+
|
234
|
+
BrotliEncoderDestroyInstance(b_state);
|
235
|
+
|
236
|
+
return (int)compressed_length;
|
237
|
+
}
|
238
|
+
|
239
|
+
static int frt_os_write_bz2_compressed_bytes(FrtOutStream* out_stream, frt_uchar *data, int length) {
|
240
|
+
int ret, buf_size, compressed_len = 0;
|
241
|
+
char out_buffer[FRT_COMPRESSION_BUFFER_SIZE];
|
242
|
+
bz_stream zstrm;
|
243
|
+
zstrm.bzalloc = NULL;
|
244
|
+
zstrm.bzfree = NULL;
|
245
|
+
zstrm.opaque = NULL;
|
246
|
+
if ((ret = BZ2_bzCompressInit(&zstrm, FRT_BZIP_COMPRESSION_LEVEL, 0, 0)) != BZ_OK) frt_zraise(ret);
|
247
|
+
|
248
|
+
zstrm.avail_in = length;
|
249
|
+
zstrm.next_in = (char *)data;
|
250
|
+
zstrm.avail_out = FRT_COMPRESSION_BUFFER_SIZE;
|
251
|
+
zstrm.next_out = out_buffer;
|
252
|
+
|
253
|
+
do {
|
254
|
+
ret = BZ2_bzCompress(&zstrm, BZ_FINISH); /* no bad return value */
|
255
|
+
assert(ret != BZ_SEQUENCE_ERROR); /* state not clobbered */
|
256
|
+
compressed_len += buf_size = FRT_COMPRESSION_BUFFER_SIZE - zstrm.avail_out;
|
257
|
+
frt_os_write_bytes(out_stream, (frt_uchar *)out_buffer, buf_size);
|
258
|
+
} while (zstrm.avail_out == 0);
|
259
|
+
assert(zstrm.avail_in == 0); /* all input will be used */
|
260
|
+
|
261
|
+
(void)BZ2_bzCompressEnd(&zstrm);
|
262
|
+
return compressed_len;
|
263
|
+
}
|
264
|
+
|
265
|
+
static const LZ4F_preferences_t lz4_prefs = {
|
266
|
+
{
|
267
|
+
LZ4F_default,
|
268
|
+
LZ4F_blockLinked,
|
269
|
+
LZ4F_noContentChecksum,
|
270
|
+
LZ4F_frame,
|
271
|
+
0, /* unknown content size */
|
272
|
+
0, /* no dictID */
|
273
|
+
LZ4F_noBlockChecksum
|
274
|
+
},
|
275
|
+
0,
|
276
|
+
1,
|
277
|
+
1,
|
278
|
+
{0,0,0}
|
279
|
+
};
|
280
|
+
|
281
|
+
static int frt_os_write_lz4_compressed_bytes(FrtOutStream* out_stream, frt_uchar *data, int length) {
|
282
|
+
int compressed_length = 0;
|
283
|
+
int remaining_length = length;
|
284
|
+
size_t ccmp_length = 0;
|
285
|
+
LZ4F_compressionContext_t ctx;
|
286
|
+
size_t out_buf_length = LZ4F_compressBound(FRT_COMPRESSION_BUFFER_SIZE, &lz4_prefs);
|
287
|
+
frt_uchar *out_buf = frt_ecalloc(out_buf_length);
|
288
|
+
|
289
|
+
size_t ctx_creation = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
|
290
|
+
if (LZ4F_isError(ctx_creation)) {
|
291
|
+
compressed_length = -1;
|
292
|
+
goto finish;
|
293
|
+
}
|
294
|
+
|
295
|
+
/* create header */
|
296
|
+
ccmp_length = LZ4F_compressBegin(ctx, out_buf, out_buf_length, &lz4_prefs);
|
297
|
+
if (LZ4F_isError(ccmp_length)) {
|
298
|
+
compressed_length = -1;
|
299
|
+
goto finish;
|
300
|
+
}
|
301
|
+
compressed_length = ccmp_length;
|
302
|
+
frt_os_write_bytes(out_stream, out_buf, ccmp_length);
|
303
|
+
|
304
|
+
/* compress data */
|
305
|
+
do {
|
306
|
+
int read_length = (FRT_COMPRESSION_BUFFER_SIZE > remaining_length) ? remaining_length : FRT_COMPRESSION_BUFFER_SIZE;
|
307
|
+
ccmp_length = LZ4F_compressUpdate(ctx, out_buf, out_buf_length, data + (length - remaining_length), read_length, NULL);
|
308
|
+
if (LZ4F_isError(ccmp_length)) {
|
309
|
+
compressed_length = -1;
|
310
|
+
goto finish;
|
311
|
+
}
|
312
|
+
frt_os_write_bytes(out_stream, out_buf, ccmp_length);
|
313
|
+
compressed_length += ccmp_length;
|
314
|
+
remaining_length -= read_length;
|
315
|
+
} while (remaining_length > 0);
|
316
|
+
|
317
|
+
/* finish up */
|
318
|
+
ccmp_length = LZ4F_compressEnd(ctx, out_buf, out_buf_length, NULL);
|
319
|
+
if (LZ4F_isError(ccmp_length)) {
|
320
|
+
compressed_length = -1;
|
321
|
+
goto finish;
|
322
|
+
}
|
323
|
+
|
324
|
+
frt_os_write_bytes(out_stream, out_buf, ccmp_length);
|
325
|
+
compressed_length += ccmp_length;
|
326
|
+
|
327
|
+
finish:
|
328
|
+
LZ4F_freeCompressionContext(ctx);
|
329
|
+
free(out_buf);
|
330
|
+
|
331
|
+
return compressed_length;
|
332
|
+
}
|
333
|
+
|
334
|
+
int frt_os_write_compressed_bytes(FrtOutStream* out_stream, frt_uchar *data, int length, FrtCompressionType compression) {
|
335
|
+
switch (compression) {
|
336
|
+
case FRT_COMPRESSION_BROTLI:
|
337
|
+
return frt_os_write_brotli_compressed_bytes(out_stream, data, length);
|
338
|
+
case FRT_COMPRESSION_BZ2:
|
339
|
+
return frt_os_write_bz2_compressed_bytes(out_stream, data, length);
|
340
|
+
case FRT_COMPRESSION_LZ4:
|
341
|
+
return frt_os_write_lz4_compressed_bytes(out_stream, data, length);
|
342
|
+
default:
|
343
|
+
return -1;
|
344
|
+
}
|
345
|
+
|
346
|
+
}
|