rroonga 5.0.4 → 5.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2 -2
- data/example/measure-data-column-disk-usage.rb +124 -0
- data/example/measure-index-column-disk-usage.rb +81 -0
- data/example/measure-table-disk-usage.rb +100 -0
- data/ext/groonga/rb-grn-database.c +31 -0
- data/ext/groonga/rb-grn-double-array-trie.c +1 -8
- data/ext/groonga/rb-grn-logger.c +45 -0
- data/ext/groonga/rb-grn-object.c +29 -1
- data/ext/groonga/rb-grn-patricia-trie.c +1 -8
- data/ext/groonga/rb-grn-table-cursor.c +8 -3
- data/ext/groonga/rb-grn-table.c +10 -5
- data/ext/groonga/rb-grn-thread.c +160 -0
- data/ext/groonga/rb-grn-windows-event-logger.c +79 -0
- data/ext/groonga/rb-grn.h +3 -1
- data/ext/groonga/rb-groonga.c +3 -1
- data/lib/groonga/dumper.rb +6 -1
- data/rroonga-build.rb +4 -4
- data/test/groonga-test-utils.rb +5 -1
- data/test/test-database.rb +11 -0
- data/test/test-logger.rb +6 -0
- data/test/test-operator.rb +6 -6
- data/test/test-procedure.rb +15 -0
- data/test/test-table-dumper.rb +170 -1
- data/test/test-thread.rb +42 -0
- data/test/test-windows-event-logger.rb +28 -0
- metadata +57 -48
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2788f17a2929324785fcf5a1de07bad9067404e0
|
4
|
+
data.tar.gz: d3b901f4819bc8a38362be54986e2ba900ded292
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 45f5bee24b2c2c1eec1a26bb37676622b53417004f08f67d36c529ac2b020eb0e7e8042b43749283b5b393ab757885b98a7bcb4f82b2e4cad6fa5d92e5380ebb
|
7
|
+
data.tar.gz: 890e882747b06939b22f7febd8a1b5976d5cfd00c055af0079295b8558dcfb3a68c6d6101cf1a9109db6a3c4a18fe4bd77cdc8579e2388ee6ccd2534ace25395
|
data/README.md
CHANGED
@@ -0,0 +1,124 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
#
|
3
|
+
# Copyright (C) 2015 Kouhei Sutou <kou@clear-code.com>
|
4
|
+
#
|
5
|
+
# This library is free software; you can redistribute it and/or
|
6
|
+
# modify it under the terms of the GNU Lesser General Public
|
7
|
+
# License version 2.1 as published by the Free Software Foundation.
|
8
|
+
#
|
9
|
+
# This library is distributed in the hope that it will be useful,
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
12
|
+
# Lesser General Public License for more details.
|
13
|
+
#
|
14
|
+
# You should have received a copy of the GNU Lesser General Public
|
15
|
+
# License along with this library; if not, write to the Free Software
|
16
|
+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
17
|
+
|
18
|
+
require "groonga"
|
19
|
+
require "fileutils"
|
20
|
+
|
21
|
+
if ARGV.size != 3
|
22
|
+
puts "Usage: #{$0} DB_DIR COLUMN_TYPE COLUMN_VALUE_TYPE"
|
23
|
+
puts " e.g.: #{$0} db scalar ShortText"
|
24
|
+
puts " e.g.: #{$0} db vector Time"
|
25
|
+
puts " e.g.: #{$0} db vector reference"
|
26
|
+
exit(false)
|
27
|
+
end
|
28
|
+
|
29
|
+
db_dir = ARGV.shift
|
30
|
+
column_type = ARGV.shift
|
31
|
+
column_value_type = ARGV.shift.to_sym
|
32
|
+
|
33
|
+
FileUtils.rm_rf(db_dir)
|
34
|
+
FileUtils.mkdir_p(db_dir)
|
35
|
+
|
36
|
+
Groonga::Database.create(:path => "#{db_dir}/db")
|
37
|
+
|
38
|
+
Groonga::Schema.define do |schema|
|
39
|
+
schema.create_table("reference",
|
40
|
+
:type => :hash,
|
41
|
+
:key_type => "Int32") do |table|
|
42
|
+
end
|
43
|
+
|
44
|
+
schema.create_table("table", :type => :array) do |table|
|
45
|
+
table.column("column", column_value_type, :type => column_type)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def measure_column_disk_usage(column)
|
50
|
+
puts "\# of records,total disk usage,increment"
|
51
|
+
table = column.domain
|
52
|
+
vector_p = column.vector?
|
53
|
+
previous_disk_usage = 0
|
54
|
+
loop do
|
55
|
+
if previous_disk_usage != column.disk_usage
|
56
|
+
diff = column.disk_usage - previous_disk_usage
|
57
|
+
puts "#{table.size},#{column.disk_usage},#{diff}"
|
58
|
+
$stdout.flush
|
59
|
+
previous_disk_usage = column.disk_usage
|
60
|
+
return if table.size > 200_000_000
|
61
|
+
end
|
62
|
+
if vector_p
|
63
|
+
value = [yield, yield, yield]
|
64
|
+
else
|
65
|
+
value = yield
|
66
|
+
end
|
67
|
+
table.add(:column => value)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def measure_reference_column_disk_usage(column)
|
72
|
+
reference_key = 0
|
73
|
+
reference = Groonga["reference"]
|
74
|
+
measure_column_disk_usage(column) do
|
75
|
+
reference_key += 1
|
76
|
+
reference.add(reference_key)
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def measure_short_text_column_disk_usage(column)
|
81
|
+
key1 = 1
|
82
|
+
key2 = 1
|
83
|
+
measure_column_disk_usage(column) do
|
84
|
+
key = "#{key1}-#{key2}"
|
85
|
+
key2 += 1
|
86
|
+
if key2 > 100
|
87
|
+
key1 += 1
|
88
|
+
key2 = 1
|
89
|
+
end
|
90
|
+
key
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
def measure_int_column_disk_usage(column)
|
95
|
+
value = 1
|
96
|
+
measure_column_disk_usage(column) do
|
97
|
+
value += 1
|
98
|
+
value
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
def measure_time_column_disk_usage(column)
|
103
|
+
value = Time.at(0)
|
104
|
+
measure_column_disk_usage(column) do
|
105
|
+
value += 1
|
106
|
+
value
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
column = Groonga["table.column"]
|
111
|
+
column_value_type_name = column.range.name
|
112
|
+
case column_value_type_name
|
113
|
+
when "reference"
|
114
|
+
measure_reference_column_disk_usage(column)
|
115
|
+
when "ShortText"
|
116
|
+
measure_short_text_column_disk_usage(column)
|
117
|
+
when /Int/
|
118
|
+
measure_int_column_disk_usage(column)
|
119
|
+
when "Time"
|
120
|
+
measure_time_column_disk_usage(column)
|
121
|
+
else
|
122
|
+
puts "Unsupported column value type: #{column_value_type_name}"
|
123
|
+
exit(false)
|
124
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
#
|
3
|
+
# Copyright (C) 2015 Kouhei Sutou <kou@clear-code.com>
|
4
|
+
#
|
5
|
+
# This library is free software; you can redistribute it and/or
|
6
|
+
# modify it under the terms of the GNU Lesser General Public
|
7
|
+
# License version 2.1 as published by the Free Software Foundation.
|
8
|
+
#
|
9
|
+
# This library is distributed in the hope that it will be useful,
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
12
|
+
# Lesser General Public License for more details.
|
13
|
+
#
|
14
|
+
# You should have received a copy of the GNU Lesser General Public
|
15
|
+
# License along with this library; if not, write to the Free Software
|
16
|
+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
17
|
+
|
18
|
+
require "groonga"
|
19
|
+
require "fileutils"
|
20
|
+
|
21
|
+
if ARGV.size != 4
|
22
|
+
puts "Usage: #{$0} DB_DIR WITH_POSITION WITH_SECTION N_POSTINGS_PER_TERM"
|
23
|
+
puts " e.g.: #{$0} db true false 100"
|
24
|
+
exit(false)
|
25
|
+
end
|
26
|
+
|
27
|
+
db_dir = ARGV.shift
|
28
|
+
with_position_p = (ARGV.shift == "true")
|
29
|
+
with_section_p = (ARGV.shift == "true")
|
30
|
+
n_postings_per_term = Integer(ARGV.shift)
|
31
|
+
|
32
|
+
FileUtils.rm_rf(db_dir)
|
33
|
+
FileUtils.mkdir_p(db_dir)
|
34
|
+
|
35
|
+
Groonga::Database.create(:path => "#{db_dir}/db")
|
36
|
+
|
37
|
+
Groonga::Schema.define do |schema|
|
38
|
+
schema.create_table("table", :type => :array) do |table|
|
39
|
+
table.column("column", "Int32")
|
40
|
+
end
|
41
|
+
|
42
|
+
schema.create_table("lexicon",
|
43
|
+
:type => :patricia_trie,
|
44
|
+
:key_type => "Int32") do |table|
|
45
|
+
table.index("table.column",
|
46
|
+
:name => "index",
|
47
|
+
:with_position => with_position_p,
|
48
|
+
:with_section => with_section_p)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def measure_disk_usage(data_table, index_column, n_postings_per_term)
|
53
|
+
puts "\# of data records,\# of terms,total disk usage,increment"
|
54
|
+
|
55
|
+
data_table = Groonga["table"]
|
56
|
+
lexicon = index_column.domain
|
57
|
+
n_postings = 0
|
58
|
+
value = 0
|
59
|
+
|
60
|
+
previous_disk_usage = 0
|
61
|
+
loop do
|
62
|
+
if previous_disk_usage != index_column.disk_usage
|
63
|
+
diff = index_column.disk_usage - previous_disk_usage
|
64
|
+
puts "#{data_table.size},#{lexicon.size},#{index_column.disk_usage},#{diff}"
|
65
|
+
$stdout.flush
|
66
|
+
previous_disk_usage = index_column.disk_usage
|
67
|
+
return if data_table.size > 200_000_000
|
68
|
+
end
|
69
|
+
if n_postings < n_postings_per_term
|
70
|
+
n_postings += 1
|
71
|
+
else
|
72
|
+
n_postings = 0
|
73
|
+
value += 1
|
74
|
+
end
|
75
|
+
data_table.add(:column => value)
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
data_table = Groonga["table"]
|
80
|
+
index_column = Groonga["lexicon.index"]
|
81
|
+
measure_disk_usage(data_table, index_column, n_postings_per_term)
|
@@ -0,0 +1,100 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
#
|
3
|
+
# Copyright (C) 2015 Kouhei Sutou <kou@clear-code.com>
|
4
|
+
#
|
5
|
+
# This library is free software; you can redistribute it and/or
|
6
|
+
# modify it under the terms of the GNU Lesser General Public
|
7
|
+
# License version 2.1 as published by the Free Software Foundation.
|
8
|
+
#
|
9
|
+
# This library is distributed in the hope that it will be useful,
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
12
|
+
# Lesser General Public License for more details.
|
13
|
+
#
|
14
|
+
# You should have received a copy of the GNU Lesser General Public
|
15
|
+
# License along with this library; if not, write to the Free Software
|
16
|
+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
17
|
+
|
18
|
+
require "groonga"
|
19
|
+
require "fileutils"
|
20
|
+
|
21
|
+
if ARGV.size != 3
|
22
|
+
puts "Usage: #{$0} DB_DIR TABLE_TYPE TABLE_KEY_TYPE"
|
23
|
+
puts " e.g.: #{$0} db hash ShortText"
|
24
|
+
exit(false)
|
25
|
+
end
|
26
|
+
|
27
|
+
db_dir = ARGV.shift
|
28
|
+
table_type = ARGV.shift
|
29
|
+
table_key_type = ARGV.shift
|
30
|
+
|
31
|
+
FileUtils.rm_rf(db_dir)
|
32
|
+
FileUtils.mkdir_p(db_dir)
|
33
|
+
|
34
|
+
Groonga::Database.create(:path => "#{db_dir}/db")
|
35
|
+
|
36
|
+
Groonga::Schema.define do |schema|
|
37
|
+
schema.create_table("table",
|
38
|
+
:type => table_type.to_sym,
|
39
|
+
:key_type => table_key_type) do
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def measure_table_disk_usage(table)
|
44
|
+
puts "\# of records,total disk usage,increment"
|
45
|
+
previous_disk_usage = 0
|
46
|
+
loop do
|
47
|
+
if previous_disk_usage != table.disk_usage
|
48
|
+
diff = table.disk_usage - previous_disk_usage
|
49
|
+
puts "#{table.size},#{table.disk_usage},#{diff}"
|
50
|
+
$stdout.flush
|
51
|
+
previous_disk_usage = table.disk_usage
|
52
|
+
return if table.size > 200_000_000
|
53
|
+
end
|
54
|
+
table.add(yield)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def measure_short_text_table_disk_usage(table)
|
59
|
+
key1 = 1
|
60
|
+
key2 = 1
|
61
|
+
measure_table_disk_usage(table) do
|
62
|
+
key = "#{key1}-#{key2}"
|
63
|
+
key2 += 1
|
64
|
+
if key2 > 100
|
65
|
+
key1 += 1
|
66
|
+
key2 = 1
|
67
|
+
end
|
68
|
+
key
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def measure_int_table_disk_usage(table)
|
73
|
+
value = 1
|
74
|
+
measure_table_disk_usage(table) do
|
75
|
+
value += 1
|
76
|
+
value
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def measure_time_table_disk_usage(table)
|
81
|
+
value = Time.at(0)
|
82
|
+
measure_table_disk_usage(table) do
|
83
|
+
value += 1
|
84
|
+
value
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
table = Groonga["table"]
|
89
|
+
table_key_type_name = table.domain.name
|
90
|
+
case table_key_type_name
|
91
|
+
when "ShortText"
|
92
|
+
measure_short_text_table_disk_usage(table)
|
93
|
+
when /Int/
|
94
|
+
measure_int_table_disk_usage(table)
|
95
|
+
when "Time"
|
96
|
+
measure_time_table_disk_usage(table)
|
97
|
+
else
|
98
|
+
puts "Unsupported table key type: #{table_key_type_name}"
|
99
|
+
exit(false)
|
100
|
+
end
|
@@ -593,6 +593,36 @@ rb_grn_database_recover (VALUE self)
|
|
593
593
|
return Qnil;
|
594
594
|
}
|
595
595
|
|
596
|
+
/*
|
597
|
+
* @overload unmap()
|
598
|
+
*
|
599
|
+
* Unmaps all mapped tables and columns in database.
|
600
|
+
*
|
601
|
+
* It frees resources for them.
|
602
|
+
*
|
603
|
+
* Normally, you don't need to unmap explicitly. Because OS manages
|
604
|
+
* resourced for mapped tables and columns cleverly.
|
605
|
+
*
|
606
|
+
* @return [void]
|
607
|
+
*
|
608
|
+
* @since 5.0.5
|
609
|
+
*/
|
610
|
+
static VALUE
|
611
|
+
rb_grn_database_unmap (VALUE self)
|
612
|
+
{
|
613
|
+
grn_rc rc;
|
614
|
+
grn_ctx *context;
|
615
|
+
grn_obj *database;
|
616
|
+
|
617
|
+
rb_grn_database_deconstruct(SELF(self), &database, &context,
|
618
|
+
NULL, NULL, NULL, NULL);
|
619
|
+
rc = grn_db_unmap(context, database);
|
620
|
+
rb_grn_context_check(context, self);
|
621
|
+
rb_grn_rc_check(rc, self);
|
622
|
+
|
623
|
+
return Qnil;
|
624
|
+
}
|
625
|
+
|
596
626
|
void
|
597
627
|
rb_grn_init_database (VALUE mGrn)
|
598
628
|
{
|
@@ -626,4 +656,5 @@ rb_grn_init_database (VALUE mGrn)
|
|
626
656
|
rb_define_method(rb_cGrnDatabase, "touch", rb_grn_database_touch, 0);
|
627
657
|
rb_define_method(rb_cGrnDatabase, "defrag", rb_grn_database_defrag, -1);
|
628
658
|
rb_define_method(rb_cGrnDatabase, "recover", rb_grn_database_recover, 0);
|
659
|
+
rb_define_method(rb_cGrnDatabase, "unmap", rb_grn_database_unmap, 0);
|
629
660
|
}
|
@@ -489,17 +489,10 @@ rb_grn_double_array_trie_open_grn_prefix_cursor (int argc, VALUE *argv,
|
|
489
489
|
} else if (rb_grn_equal_option(rb_order_by, "id")) {
|
490
490
|
flags |= GRN_CURSOR_BY_ID;
|
491
491
|
} else if (rb_grn_equal_option(rb_order_by, "key")) {
|
492
|
-
if (table->header.type != GRN_TABLE_PAT_KEY) {
|
493
|
-
rb_raise(rb_eArgError,
|
494
|
-
"order_by => :key is available "
|
495
|
-
"only for Groonga::DoubleArrayTrie: %s",
|
496
|
-
rb_grn_inspect(self));
|
497
|
-
}
|
498
492
|
flags |= GRN_CURSOR_BY_KEY;
|
499
493
|
} else {
|
500
494
|
rb_raise(rb_eArgError,
|
501
|
-
"order_by should be one of [:id
|
502
|
-
table->header.type == GRN_TABLE_PAT_KEY ? ", :key" : "",
|
495
|
+
"order_by should be one of [:id, :key]: %s",
|
503
496
|
rb_grn_inspect(rb_order_by));
|
504
497
|
}
|
505
498
|
|
data/ext/groonga/rb-grn-logger.c
CHANGED
@@ -438,6 +438,47 @@ rb_grn_logger_s_reopen (VALUE klass)
|
|
438
438
|
return Qnil;
|
439
439
|
}
|
440
440
|
|
441
|
+
/*
|
442
|
+
* @overload max_level
|
443
|
+
* @return [Symbol] The max level of the current logger.
|
444
|
+
*
|
445
|
+
* @since 5.0.5
|
446
|
+
*/
|
447
|
+
static VALUE
|
448
|
+
rb_grn_logger_s_get_max_level (VALUE klass)
|
449
|
+
{
|
450
|
+
VALUE rb_context = Qnil;
|
451
|
+
grn_ctx *context;
|
452
|
+
grn_log_level max_level;
|
453
|
+
|
454
|
+
context = rb_grn_context_ensure(&rb_context);
|
455
|
+
max_level = grn_logger_get_max_level(context);
|
456
|
+
|
457
|
+
return GRNLOGLEVEL2RVAL(max_level);
|
458
|
+
}
|
459
|
+
|
460
|
+
/*
|
461
|
+
* @overload max_level=(max_level)
|
462
|
+
* Sets the max level of the current logger.
|
463
|
+
*
|
464
|
+
* @param max_level [Symbol, String] The max level.
|
465
|
+
*
|
466
|
+
* @return [void]
|
467
|
+
*
|
468
|
+
* @since 5.0.5
|
469
|
+
*/
|
470
|
+
static VALUE
|
471
|
+
rb_grn_logger_s_set_max_level (VALUE klass, VALUE rb_max_level)
|
472
|
+
{
|
473
|
+
VALUE rb_context = Qnil;
|
474
|
+
grn_ctx *context;
|
475
|
+
|
476
|
+
context = rb_grn_context_ensure(&rb_context);
|
477
|
+
grn_logger_set_max_level(context, RVAL2GRNLOGLEVEL(rb_max_level));
|
478
|
+
|
479
|
+
return Qnil;
|
480
|
+
}
|
481
|
+
|
441
482
|
/*
|
442
483
|
* Gets the current log path that is used by the default logger.
|
443
484
|
*
|
@@ -587,6 +628,10 @@ rb_grn_init_logger (VALUE mGrn)
|
|
587
628
|
rb_grn_logger_s_unregister, 0);
|
588
629
|
rb_define_singleton_method(cGrnLogger, "reopen",
|
589
630
|
rb_grn_logger_s_reopen, 0);
|
631
|
+
rb_define_singleton_method(cGrnLogger, "max_level",
|
632
|
+
rb_grn_logger_s_get_max_level, 0);
|
633
|
+
rb_define_singleton_method(cGrnLogger, "max_level=",
|
634
|
+
rb_grn_logger_s_set_max_level, 1);
|
590
635
|
rb_define_singleton_method(cGrnLogger, "path",
|
591
636
|
rb_grn_logger_s_get_path, 0);
|
592
637
|
rb_define_singleton_method(cGrnLogger, "path=",
|