chupa-text 1.0.1 → 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: c04261f7c3b4c26a0d67f18f473c8adf1d435080
4
- data.tar.gz: 4e1a8da97abfa817226f919fcb672fb55d570d4d
3
+ metadata.gz: 5899941addcc8a58730a1d5ff4194d13f17f6504
4
+ data.tar.gz: b219d29ca910fd3a4951236f922e3bfe7211e6ef
5
5
  SHA512:
6
- metadata.gz: 00ada22d44d1f41ca4e2f08f02c43b41b193453ee33477dd02e9f5a9fbc0d87f5de46c53bc36a77b1494c2f8729a01e72419f9c9c9738d8819c04dde06c50d44
7
- data.tar.gz: 7d404268095b308d4b7158e70458ac31d2dade5a0a131431559e84b34b563fb94454d5a3e900684dfe84da2642b382a14c55a479192992b141136a5f8659faf9
6
+ metadata.gz: 8cc52887b919a6cbb4936c1c039eeb7161a239257558e01198083af19c0c4f950ee85adf21f7d3ee4dbbf0b0f06475a6484dc02633361ff9d27da33d75d2e82d
7
+ data.tar.gz: b342d42379f88c32e30a20d53008a557cca0f3e5f4ddfde75fe2aca4ad921c24bd4eef6fd0515203fec1d3f2cc71f93fc150e7bde5d43cf34055dd0da8b30d81
data/README.md CHANGED
@@ -79,6 +79,11 @@ See
79
79
  [doc/text/decomposer.md](http://rubydoc.info/gems/chupa-text/file/doc/text/decomposer.md)
80
80
  how to write a decomposer.
81
81
 
82
+ ## Available plugins
83
+
84
+ Search by `chupa-text-decomposer-` on https://rubygems.org/:
85
+ http://rubygems.org/search?query=chupa-text-decomposer-
86
+
82
87
  ## Author
83
88
 
84
89
  * Kouhei Sutou `<kou@clear-code.com>`
data/data/mime-types.conf CHANGED
@@ -19,3 +19,18 @@ mime_types["csv"] = "text/csv"
19
19
  mime_types["tsv"] = "text/tab-separated-values"
20
20
 
21
21
  mime_types["pdf"] = "application/pdf"
22
+
23
+ mime_types["odt"] = "application/vnd.oasis.opendocument.text"
24
+ mime_types["ods"] = "application/vnd.oasis.opendocument.spreadsheet"
25
+ mime_types["odp"] = "application/vnd.oasis.opendocument.presentation"
26
+
27
+ mime_types["doc"] = "application/msword"
28
+ mime_types["xls"] = "application/vnd.ms-excel"
29
+ mime_types["ppt"] = "application/vnd.ms-powerpoint"
30
+
31
+ mime_types["docx"] =
32
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
33
+ mime_types["xlsx"] =
34
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
35
+ mime_types["pptx"] =
36
+ "application/vnd.openxmlformats-officedocument.presentationml.presentation"
data/doc/text/news.md CHANGED
@@ -1,5 +1,14 @@
1
1
  # News
2
2
 
3
+ ## 1.0.2: 2014-02-15
4
+
5
+ * Added `ChupaText::SizeParser`.
6
+ * Added `ChupaText::DefaultLogger`.
7
+ * chupa-text: Added `--log-output` option.
8
+ * chupa-text: Added `--log-level` option.
9
+ * Added `ChupaText::ExternalCommand`.
10
+ * Added MIME types for office files.
11
+
3
12
  ## 1.0.1: 2014-01-05
4
13
 
5
14
  * chupa-text: Supported loading decomposers installed by RubyGems.
data/lib/chupa-text.rb CHANGED
@@ -16,15 +16,27 @@
16
16
 
17
17
  require "chupa-text/version"
18
18
 
19
+ require "chupa-text/error"
20
+
21
+ require "chupa-text/size-parser"
22
+ require "chupa-text/default-logger"
23
+ require "chupa-text/logger"
24
+
25
+ require "chupa-text/loggable"
26
+
19
27
  require "chupa-text/configuration"
20
28
  require "chupa-text/configuration-loader"
29
+ require "chupa-text/mime-type"
30
+ require "chupa-text/mime-type-registry"
31
+
32
+ require "chupa-text/external-command"
33
+
21
34
  require "chupa-text/decomposer"
22
35
  require "chupa-text/decomposer-registry"
23
36
  require "chupa-text/decomposers"
37
+
24
38
  require "chupa-text/extractor"
25
39
  require "chupa-text/formatters"
26
- require "chupa-text/mime-type"
27
- require "chupa-text/mime-type-registry"
28
40
 
29
41
  require "chupa-text/file-content"
30
42
  require "chupa-text/virtual-content"
@@ -74,21 +74,62 @@ module ChupaText
74
74
  parser = OptionParser.new
75
75
  parser.banner += " [FILE_OR_URI]"
76
76
  parser.version = VERSION
77
+
78
+ parser.separator("")
79
+ parser.separator("Generic options")
77
80
  parser.on("--configuration=FILE",
78
- "Read configuration from FILE.") do |path|
81
+ "Reads configuration from FILE.") do |path|
79
82
  load_configuration(path)
80
83
  end
81
84
  parser.on("--disable-gems",
82
- "Disable decomposers installed by RubyGems.") do
85
+ "Disables decomposers installed by RubyGems.") do
83
86
  @enable_gems = false
84
87
  end
85
88
  parser.on("-I=PATH",
86
- "Append PATH to decomposer load path.") do |path|
89
+ "Appends PATH to decomposer load path.") do |path|
87
90
  $LOAD_PATH << path
88
91
  end
92
+
93
+ parser.separator("")
94
+ parser.separator("Log related options:")
95
+ parser.on("--log-output=OUTPUT",
96
+ "Sets log output.",
97
+ "[-(stdout), +(stderr), PATH]",
98
+ "(default: +(stderr))") do |output|
99
+ ENV["CHUPA_TEXT_LOG_OUTPUT"] = output
100
+ ::ChupaText.logger = nil
101
+ end
102
+ parser.on("--log-level=LEVEL", available_log_levels,
103
+ "Sets log level.",
104
+ "[#{available_log_levels.join(', ')}]",
105
+ "(default: #{current_log_level_name})") do |level|
106
+ ENV["CHUPA_TEXT_LOG_LEVEL"] = level
107
+ ::ChupaText.logger = nil
108
+ end
109
+
89
110
  parser
90
111
  end
91
112
 
113
+ def available_log_levels
114
+ [
115
+ "debug",
116
+ "info",
117
+ "warn",
118
+ "error",
119
+ "fatal",
120
+ "unknown",
121
+ ]
122
+ end
123
+
124
+ def current_log_level_name
125
+ level = ::ChupaText.logger.level
126
+ Logger::Severity.constants.each do |name|
127
+ next if Logger::Severity.const_get(name) != level
128
+ return name.to_s.downcase
129
+ end
130
+ "info"
131
+ end
132
+
92
133
  def load_decomposers
93
134
  Decomposers.enable_all_gems if @enable_gems
94
135
  Decomposers.load
@@ -0,0 +1,150 @@
1
+ # Copyright (C) 2013 Kouhei Sutou <kou@clear-code.com>
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License as published by the Free Software Foundation; either
6
+ # version 2.1 of the License, or (at your option) any later version.
7
+ #
8
+ # This library is distributed in the hope that it will be useful,
9
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
10
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11
+ # Lesser General Public License for more details.
12
+ #
13
+ # You should have received a copy of the GNU Lesser General Public
14
+ # License along with this library; if not, write to the Free Software
15
+ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
+
17
+ require "logger"
18
+
19
+ module ChupaText
20
+ # The default logger for ChupaText. Logger options are retrieved from
21
+ # environment variables.
22
+ #
23
+ # Here are environment variables to be used:
24
+ #
25
+ # ## `CHUPA_TEXT_LOG_OUTPUT`
26
+ #
27
+ # It specifies log output.
28
+ #
29
+ # * `-`: ChupaText outputs to the standard output.
30
+ # * `+`: ChupaText outputs to the standard error output.
31
+ # * Others: It is used as file name. ChupaText outputs to the file.
32
+ #
33
+ # The default is `+` (the standard error).
34
+ #
35
+ # ## `CHUPA_TEXT_LOG_ROTATION_PERIOD`
36
+ #
37
+ # It specifies log rotation period.
38
+ #
39
+ # It is ignored when `CHUPA_TEXT_LOG_OUTPUT` is `-` (the standard
40
+ # output) or `+` (the standard error output).
41
+ #
42
+ # * `daily`: Log file is rotated daily.
43
+ # * `weekly`: Log file is rotated weekly.
44
+ # * `monthly`: Log file is rotated monthly.
45
+ # * Others: Invalid value. It is ignored.
46
+ #
47
+ # ## `CHUPA_TEXT_LOG_N_GENERATIONS`
48
+ #
49
+ # It specifies how many old log files are kept.
50
+ #
51
+ # It is ignored when (a) `CHUPA_TEXT_LOG_OUTPUT` is `-` (the
52
+ # standard output) or `+` (the standard error output) or (b)
53
+ # `CHUPA_TEXT_LOG_RATATION_PERIOD` is valid value.
54
+ #
55
+ # The default value is `7`.
56
+ #
57
+ # ## `CHUPA_TEXT_LOG_LEVEL`
58
+ #
59
+ # It specifies log verbosity.
60
+ #
61
+ # The default value is `info`.
62
+ #
63
+ # * `unknown`: ChupaText outputs only unknown messages.
64
+ # * `fatal`: ChupaText outputs `unknown` level messages and
65
+ # unhandleable error messages.
66
+ # * `error`: ChupaText outputs `fatal` level messages and
67
+ # handleable error messages.
68
+ # * `warn`: ChupaText outputs `error` level messages and warning
69
+ # level messages.
70
+ # * `info`: ChupaText outputs `warn` level messages and generic useful
71
+ # information messages.
72
+ # * `debug`: ChupaText outputs all messages.
73
+ class DefaultLogger < Logger
74
+ def initialize
75
+ super(output_device, default_shift_age, default_shift_size)
76
+ self.level = default_level
77
+ self.formatter = Formatter.new
78
+ end
79
+
80
+ private
81
+ def output_device
82
+ output = ENV["CHUPA_TEXT_LOG_OUTPUT"] || "+"
83
+ case output
84
+ when "-"
85
+ STDOUT
86
+ when "+"
87
+ STDERR
88
+ else
89
+ output
90
+ end
91
+ end
92
+
93
+ def default_shift_age
94
+ rotation_period = ENV["CHUPA_TEXT_LOG_ROTATION_PERIOD"]
95
+ case rotation_period
96
+ when "daily", "weekly", "monthly"
97
+ return rotation_period
98
+ end
99
+
100
+ n_generations = ENV["CHUPA_TEXT_LOG_N_GENERATIONS"] || "7"
101
+ begin
102
+ Integer(n_generations)
103
+ rescue ArgumentError
104
+ nil
105
+ end
106
+ end
107
+
108
+ def default_shift_size
109
+ max_size = ENV["CHUPA_TEXT_LOG_MAX_SIZE"] || "1MB"
110
+ begin
111
+ SizeParser.parse(max_size)
112
+ rescue SizeParser::InvalidSizeError
113
+ nil
114
+ end
115
+ end
116
+
117
+ def default_level
118
+ level_name = (ENV["CHUPA_TEXT_LOG_LEVEL"] || "info").upcase
119
+ if Logger::Severity.const_defined?(level_name)
120
+ Logger::Severity.const_get(level_name)
121
+ else
122
+ Logger::Severity::INFO
123
+ end
124
+ end
125
+
126
+ class Formatter
127
+ def call(severity, time, program_name, message)
128
+ "%s: [%d] %s: %s" % [
129
+ time.iso8601(6),
130
+ Process.pid,
131
+ severity[0, 1],
132
+ format_message(message),
133
+ ]
134
+ end
135
+
136
+ private
137
+ def format_message(message)
138
+ case message
139
+ when String
140
+ message
141
+ when Exception
142
+ "#{message.message}(#{message.class})\n" +
143
+ (message.backtrace || []).join("\n")
144
+ else
145
+ message.inpsect
146
+ end
147
+ end
148
+ end
149
+ end
150
+ end
@@ -0,0 +1,20 @@
1
+ # Copyright (C) 2013 Kouhei Sutou <kou@clear-code.com>
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License as published by the Free Software Foundation; either
6
+ # version 2.1 of the License, or (at your option) any later version.
7
+ #
8
+ # This library is distributed in the hope that it will be useful,
9
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
10
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11
+ # Lesser General Public License for more details.
12
+ #
13
+ # You should have received a copy of the GNU Lesser General Public
14
+ # License along with this library; if not, write to the Free Software
15
+ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
+
17
+ module ChupaText
18
+ class Error < StandardError
19
+ end
20
+ end
@@ -0,0 +1,136 @@
1
+ # Copyright (C) 2014 Kouhei Sutou <kou@clear-code.com>
2
+ # Copyright (C) 2010 Yuto HAYAMIZU <y.hayamizu@gmail.com>
3
+ #
4
+ # This library is free software; you can redistribute it and/or
5
+ # modify it under the terms of the GNU Lesser General Public
6
+ # License as published by the Free Software Foundation; either
7
+ # version 2.1 of the License, or (at your option) any later version.
8
+ #
9
+ # This library is distributed in the hope that it will be useful,
10
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12
+ # Lesser General Public License for more details.
13
+ #
14
+ # You should have received a copy of the GNU Lesser General Public
15
+ # License along with this library; if not, write to the Free Software
16
+ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
+
18
+ require "English"
19
+ require "pathname"
20
+
21
+ module ChupaText
22
+ class ExternalCommand
23
+ def initialize(command)
24
+ @command = Pathname.new(command)
25
+ end
26
+
27
+ def run(*arguments)
28
+ if arguments.last.is_a?(Hash)
29
+ options = arguments.pop
30
+ else
31
+ options = {}
32
+ end
33
+ spawn_options = options[:spawn_options] || {}
34
+ pid = spawn(options[:env] || {},
35
+ @command.to_s,
36
+ *arguments,
37
+ spawn_options.merge(default_spawn_options))
38
+ pid, status = Process.waitpid2(pid)
39
+ status.success?
40
+ end
41
+
42
+ def exist?
43
+ if @command.absolute?
44
+ @command.file? and @command.executable?
45
+ else
46
+ (ENV['PATH'] || "").split(File::PATH_SEPARATOR).any? do |path|
47
+ (Pathname.new(path) + @command).expand_path.exist?
48
+ end
49
+ end
50
+ end
51
+
52
+ private
53
+ def default_spawn_options
54
+ SpawnLimitOptions.new.options
55
+ end
56
+
57
+ class SpawnLimitOptions
58
+ attr_reader :options
59
+ def initialize
60
+ @options = {}
61
+ set_default_options
62
+ end
63
+
64
+ private
65
+ def set_default_options
66
+ set_option(:cpu, :int)
67
+ set_option(:rss, :size)
68
+ set_option(:as, :size)
69
+ end
70
+
71
+ def set_option(key, type)
72
+ value = ENV["CHUPA_EXTERNAL_COMMAND_LIMIT_#{key.to_s.upcase}"]
73
+ return if value.nil?
74
+ value = send("parse_#{type}", key, value)
75
+ return if value.nil?
76
+ rlimit_number = Process.const_get("RLIMIT_#{key.to_s.upcase}")
77
+ soft_limit, hard_limit = Process.getrlimit(rlimit_number)
78
+ if hard_limit < value
79
+ log_hard_limit_over_value(key, value, hard_limit)
80
+ return nil
81
+ end
82
+ limit_info = "soft-limit:#{soft_limit}, hard-limit:#{hard_limit}"
83
+ log(:info, "[#{key}][set] <#{value}>(#{limit_info})")
84
+ @options[:"rlimit_#{key}"] = value
85
+ end
86
+
87
+ def parse_int(key, value)
88
+ begin
89
+ Integer(value)
90
+ rescue ArgumentError
91
+ log_invalid_value(key, value, type, "int")
92
+ nil
93
+ end
94
+ end
95
+
96
+ def parse_size(key, value)
97
+ return nil if value.nil?
98
+ scale = 1
99
+ case value
100
+ when /GB?\z/i
101
+ scale = 1024 ** 3
102
+ number = $PREMATCH
103
+ when /MB?\z/i
104
+ scale = 1024 ** 2
105
+ number = $PREMATCH
106
+ when /KB?\z/i
107
+ scale = 1024 ** 1
108
+ number = $PREMATCH
109
+ when /B?\z/i
110
+ number = $PREMATCH
111
+ else
112
+ number = value
113
+ end
114
+ begin
115
+ number = Float(number)
116
+ rescue ArgumentError
117
+ log_invalid_value(key, value, "size")
118
+ return nil
119
+ end
120
+ (number * scale).to_i
121
+ end
122
+
123
+ def log_hard_limit_over_value(key, value, hard_limit)
124
+ log(:warning, "[#{key}][large] <#{value}>(hard-limit:#{hard_limit})")
125
+ end
126
+
127
+ def log_invalid_value(key, value, type)
128
+ log(:warning, "[#{key}][invalid] <#{value}>(#{type})")
129
+ end
130
+
131
+ def log(level, message)
132
+ ChupaText.logger.send(level, "[external-command][limit]#{message}")
133
+ end
134
+ end
135
+ end
136
+ end
@@ -19,6 +19,8 @@ require "uri"
19
19
 
20
20
  module ChupaText
21
21
  class Extractor
22
+ include Loggable
23
+
22
24
  def initialize
23
25
  @decomposers = []
24
26
  end
@@ -59,16 +61,27 @@ module ChupaText
59
61
  targets = [ensure_data(input)]
60
62
  until targets.empty?
61
63
  target = targets.pop
64
+ debug do
65
+ "#{log_tag}[extract][target] <#{target.path}>:<#{target.mime_type}>"
66
+ end
62
67
  if target.text_plain?
63
68
  yield(target)
64
69
  next
65
70
  end
66
71
  decomposer = find_decomposer(target)
67
72
  if decomposer.nil?
73
+ debug {"#{log_tag}[extract][decomposer] not found"}
68
74
  yield(target) if target.text?
69
75
  next
70
76
  end
77
+ debug {"#{log_tag}[extract][decomposer] #{decomposer.class}"}
71
78
  decomposer.decompose(target) do |decomposed|
79
+ debug do
80
+ "#{log_tag}[extract][decomposed] " +
81
+ "#{decomposer.class}: " +
82
+ "<#{target.path}>:<#{target.mime_type}> -> " +
83
+ "<#{decomposed.mime_type}>"
84
+ end
72
85
  targets.push(decomposed)
73
86
  end
74
87
  end
@@ -88,5 +101,9 @@ module ChupaText
88
101
  decomposer.target?(data)
89
102
  end
90
103
  end
104
+
105
+ def log_tag
106
+ "[extractor]"
107
+ end
91
108
  end
92
109
  end
@@ -0,0 +1,49 @@
1
+ # Copyright (C) 2013 Kouhei Sutou <kou@clear-code.com>
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License as published by the Free Software Foundation; either
6
+ # version 2.1 of the License, or (at your option) any later version.
7
+ #
8
+ # This library is distributed in the hope that it will be useful,
9
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
10
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11
+ # Lesser General Public License for more details.
12
+ #
13
+ # You should have received a copy of the GNU Lesser General Public
14
+ # License along with this library; if not, write to the Free Software
15
+ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
+
17
+ module ChupaText
18
+ # Adds shortcut methods for easy to log.
19
+ module Loggable
20
+ private
21
+ def logger
22
+ ChupaText.logger
23
+ end
24
+
25
+ def debug(*arguments, &block)
26
+ logger.debug(*arguments, &block)
27
+ end
28
+
29
+ def info(*arguments, &block)
30
+ logger.info(*arguments, &block)
31
+ end
32
+
33
+ def warn(*arguments, &block)
34
+ logger.warn(*arguments, &block)
35
+ end
36
+
37
+ def error(*arguments, &block)
38
+ logger.error(*arguments, &block)
39
+ end
40
+
41
+ def fatal(*arguments, &block)
42
+ logger.fatal(*arguments, &block)
43
+ end
44
+
45
+ def unknown(*arguments, &block)
46
+ logger.unknown(*arguments, &block)
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,30 @@
1
+ # Copyright (C) 2013 Kouhei Sutou <kou@clear-code.com>
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License as published by the Free Software Foundation; either
6
+ # version 2.1 of the License, or (at your option) any later version.
7
+ #
8
+ # This library is distributed in the hope that it will be useful,
9
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
10
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11
+ # Lesser General Public License for more details.
12
+ #
13
+ # You should have received a copy of the GNU Lesser General Public
14
+ # License along with this library; if not, write to the Free Software
15
+ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
+
17
+ module ChupaText
18
+ class << self
19
+ # @return [Logger] The logger of ChupaText.
20
+ # {ChupaText::DefaultLogger} is used by default.
21
+ def logger
22
+ @@logger ||= DefaultLogger.new
23
+ end
24
+
25
+ # @param [Logger] logger The logger of ChupaText.
26
+ def logger=(logger)
27
+ @@logger = logger
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,65 @@
1
+ # Copyright (C) 2013 Kouhei Sutou <kou@clear-code.com>
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License as published by the Free Software Foundation; either
6
+ # version 2.1 of the License, or (at your option) any later version.
7
+ #
8
+ # This library is distributed in the hope that it will be useful,
9
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
10
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11
+ # Lesser General Public License for more details.
12
+ #
13
+ # You should have received a copy of the GNU Lesser General Public
14
+ # License along with this library; if not, write to the Free Software
15
+ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
+
17
+ require "English"
18
+
19
+ module ChupaText
20
+ class SizeParser
21
+ class InvalidSizeError < Error
22
+ attr_reader :size
23
+ def initialize(size)
24
+ @size = size
25
+ super("invalid size: <#{@size.inspect}>")
26
+ end
27
+ end
28
+
29
+ class << self
30
+ def parse(size)
31
+ new.parse(size)
32
+ end
33
+ end
34
+
35
+ def parse(size)
36
+ case size
37
+ when /TB?\z/i
38
+ scale = 1024 ** 4
39
+ number_part = $PREMATCH
40
+ when /GB?\z/i
41
+ scale = 1024 ** 3
42
+ number_part = $PREMATCH
43
+ when /MB?\z/i
44
+ scale = 1024 ** 2
45
+ number_part = $PREMATCH
46
+ when /KB?\z/i
47
+ scale = 1024 ** 1
48
+ number_part = $PREMATCH
49
+ when /B?\z/i
50
+ scale = 1
51
+ number_part = $PREMATCH
52
+ else
53
+ scale = 1
54
+ number_part = size
55
+ end
56
+
57
+ begin
58
+ number = Float(number_part)
59
+ rescue ArgumentError
60
+ raise InvalidSizeError.new(size)
61
+ end
62
+ (number * scale).round
63
+ end
64
+ end
65
+ end
@@ -15,5 +15,5 @@
15
15
  # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
16
 
17
17
  module ChupaText
18
- VERSION = "1.0.1"
18
+ VERSION = "1.0.2"
19
19
  end
@@ -0,0 +1,184 @@
1
+ # Copyright (C) 2013 Kouhei Sutou <kou@clear-code.com>
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License as published by the Free Software Foundation; either
6
+ # version 2.1 of the License, or (at your option) any later version.
7
+ #
8
+ # This library is distributed in the hope that it will be useful,
9
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
10
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11
+ # Lesser General Public License for more details.
12
+ #
13
+ # You should have received a copy of the GNU Lesser General Public
14
+ # License along with this library; if not, write to the Free Software
15
+ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
+
17
+ class TestDefaultLogger < Test::Unit::TestCase
18
+ def setup
19
+ @env = {}
20
+ ENV.each do |key, value|
21
+ @env[key] = value
22
+ end
23
+ end
24
+
25
+ def teardown
26
+ ENV.replace(@env)
27
+ end
28
+
29
+ private
30
+ def default_n_generations
31
+ 7
32
+ end
33
+
34
+ def default_max_size
35
+ 1024 * 1024
36
+ end
37
+
38
+ sub_test_case("output") do
39
+ def output(value)
40
+ ENV["CHUPA_TEXT_LOG_OUTPUT"] = value
41
+ logger = ChupaText::DefaultLogger.new
42
+ device = logger.instance_variable_get(:@logdev)
43
+ device.dev
44
+ end
45
+
46
+ def test_minus
47
+ assert_equal(STDOUT, output("-"))
48
+ end
49
+
50
+ def test_plus
51
+ assert_equal(STDERR, output("+"))
52
+ end
53
+
54
+ def test_string
55
+ file = Tempfile.new("chupa-text-default-logger-output")
56
+ assert_equal(file.path, output(file.path).path)
57
+ end
58
+
59
+ def test_default
60
+ assert_equal(STDERR, output(nil))
61
+ end
62
+ end
63
+
64
+ sub_test_case("rotation period") do
65
+ def rotation_period(value)
66
+ file = Tempfile.new("chupa-text-default-logger-output")
67
+ ENV["CHUPA_TEXT_LOG_OUTPUT"] = file.path
68
+ ENV["CHUPA_TEXT_LOG_ROTATION_PERIOD"] = value
69
+ logger = ChupaText::DefaultLogger.new
70
+ device = logger.instance_variable_get(:@logdev)
71
+ device.instance_variable_get(:@shift_age)
72
+ end
73
+
74
+ def test_daily
75
+ assert_equal("daily", rotation_period("daily"))
76
+ end
77
+
78
+ def test_weekly
79
+ assert_equal("weekly", rotation_period("weekly"))
80
+ end
81
+
82
+ def test_monthly
83
+ assert_equal("monthly", rotation_period("monthly"))
84
+ end
85
+
86
+ def test_default
87
+ assert_equal(default_n_generations, rotation_period(nil))
88
+ end
89
+
90
+ def test_invalid
91
+ assert_equal(default_n_generations, rotation_period(nil))
92
+ end
93
+ end
94
+
95
+ sub_test_case("N generation") do
96
+ def n_generations(value)
97
+ file = Tempfile.new("chupa-text-default-logger-output")
98
+ ENV["CHUPA_TEXT_LOG_OUTPUT"] = file.path
99
+ ENV["CHUPA_TEXT_LOG_N_GENERATIONS"] = value
100
+ logger = ChupaText::DefaultLogger.new
101
+ device = logger.instance_variable_get(:@logdev)
102
+ device.instance_variable_get(:@shift_age)
103
+ end
104
+
105
+ def test_integer
106
+ assert_equal(29, n_generations("29"))
107
+ end
108
+
109
+ def test_default
110
+ assert_equal(default_n_generations, n_generations(nil))
111
+ end
112
+
113
+ def test_invalid
114
+ assert_equal(default_n_generations, n_generations("2.9"))
115
+ end
116
+ end
117
+
118
+ sub_test_case("max size") do
119
+ def max_size(value)
120
+ file = Tempfile.new("chupa-text-default-logger-output")
121
+ ENV["CHUPA_TEXT_LOG_OUTPUT"] = file.path
122
+ ENV["CHUPA_TEXT_LOG_MAX_SIZE"] = value
123
+ logger = ChupaText::DefaultLogger.new
124
+ device = logger.instance_variable_get(:@logdev)
125
+ device.instance_variable_get(:@shift_size)
126
+ end
127
+
128
+ def test_unit
129
+ assert_equal(1024, max_size("1KB"))
130
+ end
131
+
132
+ def test_value_only
133
+ assert_equal(1024, max_size("1024"))
134
+ end
135
+
136
+ def test_default
137
+ assert_equal(default_max_size, max_size(nil))
138
+ end
139
+
140
+ def test_invalid
141
+ assert_equal(default_max_size, max_size("max-size"))
142
+ end
143
+ end
144
+
145
+ sub_test_case("level") do
146
+ def level(value)
147
+ ENV["CHUPA_TEXT_LOG_LEVEL"] = value
148
+ logger = ChupaText::DefaultLogger.new
149
+ logger.level
150
+ end
151
+
152
+ def test_debug
153
+ assert_equal(Logger::DEBUG, level("debug"))
154
+ end
155
+
156
+ def test_info
157
+ assert_equal(Logger::INFO, level("info"))
158
+ end
159
+
160
+ def test_warn
161
+ assert_equal(Logger::WARN, level("warn"))
162
+ end
163
+
164
+ def test_error
165
+ assert_equal(Logger::ERROR, level("error"))
166
+ end
167
+
168
+ def test_fatal
169
+ assert_equal(Logger::FATAL, level("fatal"))
170
+ end
171
+
172
+ def test_unknown
173
+ assert_equal(Logger::UNKNOWN, level("unknown"))
174
+ end
175
+
176
+ def test_default
177
+ assert_equal(Logger::INFO, level(nil))
178
+ end
179
+
180
+ def test_invalid
181
+ assert_equal(Logger::INFO, level("invalid"))
182
+ end
183
+ end
184
+ end
@@ -0,0 +1,79 @@
1
+ # Copyright (C) 2014 Kouhei Sutou <kou@clear-code.com>
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License as published by the Free Software Foundation; either
6
+ # version 2.1 of the License, or (at your option) any later version.
7
+ #
8
+ # This library is distributed in the hope that it will be useful,
9
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
10
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11
+ # Lesser General Public License for more details.
12
+ #
13
+ # You should have received a copy of the GNU Lesser General Public
14
+ # License along with this library; if not, write to the Free Software
15
+ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
+
17
+ require "rbconfig"
18
+
19
+ class TestExternalCommand < Test::Unit::TestCase
20
+ def ruby
21
+ RbConfig.ruby
22
+ end
23
+
24
+ def create_command(command)
25
+ ChupaText::ExternalCommand.new(command)
26
+ end
27
+
28
+ class TestRun < self
29
+ def run_command(command, *arguments)
30
+ create_command(command).run(*arguments)
31
+ end
32
+
33
+ def test_success
34
+ assert_true(run_command(ruby, "-e", "true"))
35
+ end
36
+
37
+ def test_failure
38
+ error = Tempfile.new("error")
39
+ spawn_options = {
40
+ :err => error.path,
41
+ }
42
+ assert_false(run_command(ruby,
43
+ "-e", "raise 'XXX'",
44
+ :spawn_options => spawn_options))
45
+ end
46
+ end
47
+
48
+ class TestExist < self
49
+ def setup
50
+ @original_path = ENV["PATH"]
51
+ end
52
+
53
+ def teardown
54
+ ENV["PATH"] = @original_path
55
+ end
56
+
57
+ def exist?(command)
58
+ create_command(command).exist?
59
+ end
60
+
61
+ def test_exist_absolete_path
62
+ assert_true(exist?(ruby))
63
+ end
64
+
65
+ def test_exist_in_path
66
+ ruby_dir, ruby_base_name = File.split(ruby)
67
+ ENV["PATH"] += "#{File::PATH_SEPARATOR}#{ruby_dir}"
68
+ assert_true(exist?(ruby_base_name))
69
+ end
70
+
71
+ def test_not_executable
72
+ assert_false(exist?(__FILE__))
73
+ end
74
+
75
+ def test_not_exist
76
+ assert_false(exist?("nonexistent"))
77
+ end
78
+ end
79
+ end
@@ -0,0 +1,79 @@
1
+ # Copyright (C) 2013 Kouhei Sutou <kou@clear-code.com>
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License as published by the Free Software Foundation; either
6
+ # version 2.1 of the License, or (at your option) any later version.
7
+ #
8
+ # This library is distributed in the hope that it will be useful,
9
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
10
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11
+ # Lesser General Public License for more details.
12
+ #
13
+ # You should have received a copy of the GNU Lesser General Public
14
+ # License along with this library; if not, write to the Free Software
15
+ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
+
17
+ class TestSizeParser < Test::Unit::TestCase
18
+ private
19
+ def parse(value)
20
+ ChupaText::SizeParser.parse(value)
21
+ end
22
+
23
+ sub_test_case("unit") do
24
+ def test_terabyte
25
+ assert_equal(1024 ** 4, parse("1TB"))
26
+ end
27
+
28
+ def test_tera
29
+ assert_equal(1024 ** 4, parse("1T"))
30
+ end
31
+
32
+ def test_gigabyte
33
+ assert_equal(1024 ** 3, parse("1GB"))
34
+ end
35
+
36
+ def test_giga
37
+ assert_equal(1024 ** 3, parse("1G"))
38
+ end
39
+
40
+ def test_megabyte
41
+ assert_equal(1024 ** 2, parse("1MB"))
42
+ end
43
+
44
+ def test_mega
45
+ assert_equal(1024 ** 2, parse("1M"))
46
+ end
47
+
48
+ def test_kilobyte
49
+ assert_equal(1024 ** 1, parse("1KB"))
50
+ end
51
+
52
+ def test_kilo
53
+ assert_equal(1024 ** 1, parse("1K"))
54
+ end
55
+
56
+ def test_byte
57
+ assert_equal(1, parse("1B"))
58
+ end
59
+ end
60
+
61
+ sub_test_case("float") do
62
+ def test_with_unit
63
+ assert_equal(1024 + 512, parse("1.5KB"))
64
+ end
65
+
66
+ def test_without_unit
67
+ assert_equal(2, parse("1.5"))
68
+ end
69
+ end
70
+
71
+ sub_test_case("invalid") do
72
+ def test_unknwon_unit
73
+ size = "1.5PB"
74
+ assert_raise(ChupaText::SizeParser::InvalidSizeError.new(size)) do
75
+ parse(size)
76
+ end
77
+ end
78
+ end
79
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: chupa-text
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.1
4
+ version: 1.0.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kouhei Sutou
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-01-05 00:00:00.000000000 Z
11
+ date: 2014-02-16 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -99,6 +99,7 @@ files:
99
99
  - data/mime-types.conf
100
100
  - lib/chupa-text/data.rb
101
101
  - lib/chupa-text/version.rb
102
+ - lib/chupa-text/logger.rb
102
103
  - lib/chupa-text/extractor.rb
103
104
  - lib/chupa-text/command/chupa-text.rb
104
105
  - lib/chupa-text/command/chupa-text-generate-decomposer.rb
@@ -107,26 +108,32 @@ files:
107
108
  - lib/chupa-text/text-data.rb
108
109
  - lib/chupa-text/configuration-loader.rb
109
110
  - lib/chupa-text/virtual-file-data.rb
111
+ - lib/chupa-text/size-parser.rb
110
112
  - lib/chupa-text/formatters/json.rb
111
113
  - lib/chupa-text/virtual-content.rb
112
114
  - lib/chupa-text/decomposers/csv.rb
113
115
  - lib/chupa-text/decomposers/tar.rb
114
116
  - lib/chupa-text/decomposers/xml.rb
115
117
  - lib/chupa-text/decomposers/gzip.rb
118
+ - lib/chupa-text/error.rb
116
119
  - lib/chupa-text/formatters.rb
117
120
  - lib/chupa-text/decomposers.rb
118
121
  - lib/chupa-text/command.rb
119
122
  - lib/chupa-text/file-content.rb
120
123
  - lib/chupa-text/decomposer.rb
124
+ - lib/chupa-text/external-command.rb
121
125
  - lib/chupa-text/mime-type.rb
126
+ - lib/chupa-text/loggable.rb
122
127
  - lib/chupa-text/input-data.rb
123
128
  - lib/chupa-text/mime-type-registry.rb
129
+ - lib/chupa-text/default-logger.rb
124
130
  - lib/chupa-text.rb
125
131
  - doc/text/decomposer.md
126
132
  - doc/text/command-line.md
127
133
  - doc/text/news.md
128
134
  - doc/text/library.md
129
135
  - test/test-decomposers.rb
136
+ - test/test-default-logger.rb
130
137
  - test/test-decomposer.rb
131
138
  - test/test-virtual-content.rb
132
139
  - test/command/test-chupa-text.rb
@@ -143,6 +150,8 @@ files:
143
150
  - test/fixture/command/chupa-text/no-decomposer.conf
144
151
  - test/fixture/command/chupa-text/hello.txt
145
152
  - test/fixture/extractor/hello.txt
153
+ - test/test-external-command.rb
154
+ - test/test-size-parser.rb
146
155
  - test/decomposers/test-csv.rb
147
156
  - test/decomposers/test-gzip.rb
148
157
  - test/decomposers/test-tar.rb