yomikomu 0.3.0 → 0.4.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 547fca221d04dc83b57b2aa5dfe64d02c0707a85
4
- data.tar.gz: 3b69bf15676c15ddbc0aacd588ab34817e7a63c8
3
+ metadata.gz: 6e1bcc1dbf78cadc55edd354c6a420358558e084
4
+ data.tar.gz: da90ed8f437df5aadbbb6b10b73e9f783b51b2aa
5
5
  SHA512:
6
- metadata.gz: 822c25e871fefe58dcdcbed67739fcf9157d59e8011981bff9c3ea3b51f3c47d5fe7c01db9397076035716038c7ffeb96640e5b41c69fea443b26a4669a66883
7
- data.tar.gz: 86db6de8c2ac86e679fde45a4ef1de8f4f48bc406adf55e7fe64a568d5cca0c6e128d38f7d05f49651d1dc72832eea43cb9db28dccd4f9e903eb24f46e158413
6
+ metadata.gz: 4a4b925c4c284dc225ad5d92905685d7ee48d5daad135acf05d5c2a89949023ae43fb52dbc50f3574ccda99e53eebd8ca73d7ad6b3c3c81bf56944c7b447a555
7
+ data.tar.gz: 0302472fcbb4dd4f0e78d040998ce0302df08e5debaef057cd9a5f8bd17c9e1adb12433c5f93fb4cc4bc29efb36aaa0a769253c132e844c40ee79266403aebda
data/README.md CHANGED
@@ -31,9 +31,13 @@ You can use the following configuration with environment variables.
31
31
  * YOMIKOMU_STORAGE (default: fs): choose storage type.
32
32
  * 'fs' (default) stores binaries in the same directory (for examlple, `x.rb` will be compiled to `x.rb.yarb` in the same directory).
33
33
  * 'fs2' stores binaries in specific directory.
34
+ * 'fsgz' stores files same as 'fs', but gz compressed.
35
+ * 'fs2gz' stores files same as 'fs2', but gz compressed.
34
36
  * 'dbm' stores binaries using dbm
37
+ * 'flatfile' stores binaries in one flat file. Updating is not supported. Use with YOMIKOMU_AUTO_COMPILE.
35
38
  * YOMIKOMU_STORAGE_DIR (default: "~/.ruby_binaries"): choose directory where binary files are stored.
36
39
  * YOMIKOMU_AUTO_COMPILE (default: false): if this value is `true`, then compile all required scripts implicitly.
40
+ * YOMIKOMU_INFO (default: not defaind): show some information.
37
41
  * YOMIKOMU_DEBUG (default: not defined): show many information.
38
42
 
39
43
  ### Compile and store instruction sequences
data/exe/kakidasu CHANGED
@@ -2,6 +2,8 @@
2
2
 
3
3
  $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
4
4
 
5
+ require 'yomikomu'
6
+ require 'rbconfig'
5
7
  require 'optparse'
6
8
 
7
9
  verbose = false
@@ -37,8 +39,6 @@ executor = Proc.new{|file|
37
39
 
38
40
  opts.parse!(ARGV)
39
41
 
40
- require 'yomikomu'
41
- require 'rbconfig'
42
42
  paths = ARGV.empty? ? [RbConfig::CONFIG['libdir']] : ARGV
43
43
 
44
44
  paths.each{|path|
data/lib/yomikomu.rb CHANGED
@@ -3,22 +3,23 @@ require "yomikomu/version"
3
3
  module Yomikomu
4
4
  STATISTICS = Hash.new(0)
5
5
 
6
- unless yomu_dir = ENV['YOMIKOMU_STORAGE_DIR']
7
- yomu_dir = File.expand_path("~/.ruby_binaries")
8
- unless File.exist?(yomu_dir)
9
- Dir.mkdir(yomu_dir)
6
+ def self.prefix
7
+ unless yomu_dir = ENV['YOMIKOMU_STORAGE_DIR']
8
+ yomu_dir = File.expand_path("~/.ruby_binaries")
10
9
  end
10
+ Dir.mkdir(yomu_dir) unless File.exist?(yomu_dir)
11
+ "#{yomu_dir}/cb."
11
12
  end
12
- YOMIKOMU_PREFIX = "#{yomu_dir}/cb."
13
+
13
14
  YOMIKOMU_AUTO_COMPILE = ENV['YOMIKOMU_AUTO_COMPILE'] == 'true'
14
15
  YOMIKOMU_USE_MMAP = ENV['YOMIKOMU_USE_MMAP']
15
16
 
16
17
  def self.status
17
18
  STDERR.puts "[YOMIKOMU:INFO] (pid:#{Process.pid}) " +
18
- ::Yomikomu::STATISTICS.map{|k, v| "#{k}: #{v}"}.join(' ,')
19
+ ::Yomikomu::STATISTICS.map{|k, v| "#{k}: #{v}"}.join(', ')
19
20
  end
20
21
 
21
- if $VERBOSE
22
+ if $VERBOSE || ENV['YOMIKOMU_INFO'] == 'true'
22
23
  def self.info
23
24
  STDERR.puts "[YOMIKOMU:INFO] (pid:#{Process.pid}) #{yield}"
24
25
  end
@@ -74,14 +75,18 @@ module Yomikomu
74
75
  "SHA-1:#{::Digest::SHA1.file(fname).digest}"
75
76
  end
76
77
 
77
- def compile_and_store_iseq fname, iseq_key = iseq_key_name(fname)
78
- ::Yomikomu::STATISTICS[:compiled] += 1
78
+ def compile_and_store_iseq fname, iseq_key = iseq_key_name(fname = File.expand_path(fname))
79
79
  ::Yomikomu.debug{ "compile #{fname} into #{iseq_key}" }
80
- iseq = RubyVM::InstructionSequence.compile_file(fname)
81
-
82
- binary = iseq.to_binary(extra_data(fname))
83
- write_compiled_iseq(fname, iseq_key, binary)
84
- iseq
80
+ begin
81
+ iseq = RubyVM::InstructionSequence.compile_file(fname)
82
+ binary = iseq.to_binary(extra_data(fname))
83
+ write_compiled_iseq(fname, iseq_key, binary)
84
+ ::Yomikomu::STATISTICS[:compiled] += 1
85
+ iseq
86
+ rescue SyntaxError, RuntimeError => e
87
+ puts "#{e}: #{fname}"
88
+ nil
89
+ end
85
90
  end
86
91
 
87
92
  # def remove_compiled_iseq fname; nil; end # should implement at sub classes
@@ -102,11 +107,6 @@ module Yomikomu
102
107
  class FSStorage < BasicStorage
103
108
  def initialize
104
109
  super
105
- require 'fileutils'
106
- @dir = YOMIKOMU_PREFIX + "files"
107
- unless File.directory?(@dir)
108
- FileUtils.mkdir_p(@dir)
109
- end
110
110
  end
111
111
 
112
112
  def remove_compiled_iseq fname
@@ -144,7 +144,21 @@ module Yomikomu
144
144
  end
145
145
  end
146
146
 
147
+ class FSSGZtorage < FSStorage
148
+
149
+ end
150
+
147
151
  class FS2Storage < FSStorage
152
+ def initialize
153
+ super
154
+
155
+ require 'fileutils'
156
+ @dir = Yomikomu.prefix + "files"
157
+ unless File.directory?(@dir)
158
+ FileUtils.mkdir_p(@dir)
159
+ end
160
+ end
161
+
148
162
  def iseq_key_name fname
149
163
  File.join(@dir, fname.gsub(/[^A-Za-z0-9\._-]/){|c| '%02x' % c.ord} + '.yarb') # special directory
150
164
  end
@@ -157,6 +171,37 @@ module Yomikomu
157
171
  end
158
172
  end
159
173
 
174
+ module GZFileStorage
175
+ def initialize
176
+ require 'zlib'
177
+ super
178
+ end
179
+
180
+ def iseq_key_name fname
181
+ super + '.gz'
182
+ end
183
+
184
+ def read_compiled_iseq fname, iseq_key
185
+ Zlib::GzipReader.open(iseq_key){|f|
186
+ f.read
187
+ }
188
+ end
189
+
190
+ def write_compiled_iseq fname, iseq_key, binary
191
+ Zlib::GzipWriter.open(iseq_key){|f|
192
+ f.write(binary)
193
+ }
194
+ end
195
+ end
196
+
197
+ class FSGZStorage < FSStorage
198
+ include GZFileStorage
199
+ end
200
+
201
+ class FS2GZStorage < FS2Storage
202
+ include GZFileStorage
203
+ end
204
+
160
205
  if YOMIKOMU_USE_MMAP
161
206
  require 'mmapped_string'
162
207
  Yomikomu.info{ "[RUBY_YOMIKOMU] use mmap" }
@@ -178,8 +223,9 @@ module Yomikomu
178
223
 
179
224
  class DBMStorage < BasicStorage
180
225
  def initialize
226
+ super
181
227
  require 'dbm'
182
- @db = DBM.open(YOMIKOMU_PREFIX+'db')
228
+ @db = DBM.open(Yomikomu.prefix + 'db')
183
229
  end
184
230
 
185
231
  def remove_compiled_iseq fname
@@ -218,6 +264,100 @@ module Yomikomu
218
264
  end
219
265
  end
220
266
 
267
+ class FlatFileStorage < BasicStorage
268
+ def index_path
269
+ Yomikomu.prefix + 'ff_index'
270
+ end
271
+
272
+ def data_path
273
+ Yomikomu.prefix + 'ff_data'
274
+ end
275
+
276
+ def initialize
277
+ super
278
+ require 'fileutils'
279
+
280
+ @updated = false
281
+
282
+ if File.exist?(index_path)
283
+ open(index_path, 'rb'){|f| @index = Marshal.load(f)}
284
+ else
285
+ @index = {}
286
+ open(data_path, 'w'){} # touch
287
+ end
288
+
289
+ @data_file = open(data_path, 'a+b')
290
+
291
+ at_exit{
292
+ if @updated
293
+ open(index_path, 'wb'){|f| Marshal.dump(@index, f)}
294
+ Yomikomu.info{'FlatFile: update'}
295
+ end
296
+ }
297
+ end
298
+
299
+ def remove_compiled_iseq fname
300
+ raise 'unsupported'
301
+ end
302
+
303
+ private
304
+
305
+ def compiled_iseq_exist? fname, iseq_key
306
+ @index[iseq_key]
307
+ end
308
+
309
+ def compiled_iseq_is_younger? fname, iseq_key
310
+ offset, size, date = @index[iseq_key]
311
+ date.to_i >= File.mtime(fname).to_i
312
+ end
313
+
314
+ def read_compiled_iseq fname, iseq_key
315
+ offset, size, date = @index[iseq_key]
316
+ @data_file.pos = offset
317
+ binary = @data_file.read(size)
318
+ raise "size is not match" if binary.size != size
319
+ binary
320
+ end
321
+
322
+ def write_compiled_iseq fname, iseq_key, binary
323
+ raise "compiled binary for #{fname} already exists. flatfile does not support overwrite." if compiled_iseq_exist?(fname, iseq_key)
324
+
325
+ @data_file.seek 0, IO::SEEK_END
326
+ offset = @data_file.tell
327
+ size = binary.size
328
+ date = Time.now.to_i
329
+ @data_file.write(binary)
330
+ @index[iseq_key] = [offset, size, date]
331
+
332
+ @updated = true
333
+ end
334
+ end
335
+
336
+ class FlatFileGZStorage < FlatFileStorage
337
+ def index_path
338
+ super + '_gz'
339
+ end
340
+
341
+ def data_path
342
+ super + '_gz'
343
+ end
344
+
345
+ def initialize
346
+ super
347
+ require 'zlib'
348
+ end
349
+
350
+ def read_compiled_iseq fname, iseq_key
351
+ binary = super
352
+ Zlib::Inflate.inflate(binary)
353
+ end
354
+
355
+ def write_compiled_iseq fname, iseq_key, binary
356
+ binary = Zlib::Deflate.deflate(binary)
357
+ super(fname, iseq_key, binary)
358
+ end
359
+ end
360
+
221
361
  def self.compile_and_store_iseq fname
222
362
  STORAGE.compile_and_store_iseq fname
223
363
  end
@@ -235,17 +375,27 @@ module Yomikomu
235
375
  end
236
376
 
237
377
  # select storage
238
- STORAGE = case ENV['YOMIKOMU_STORAGE']
239
- when 'dbm'
240
- DBMStorage.new
378
+ STORAGE = case storage = ENV['YOMIKOMU_STORAGE']
241
379
  when 'fs'
242
380
  FSStorage.new
381
+ when 'fsgz'
382
+ FSGZStorage.new
243
383
  when 'fs2'
244
384
  FS2Storage.new
385
+ when 'fs2gz'
386
+ FS2GZStorage.new
387
+ when 'dbm'
388
+ DBMStorage.new
389
+ when 'flatfile'
390
+ FlatFileStorage.new
391
+ when 'flatfilegz'
392
+ FlatFileGZStorage.new
245
393
  when 'null'
246
394
  NullStorage.new
247
- else
395
+ when nil
248
396
  FSStorage.new
397
+ else
398
+ raise "Unknown storage type: #{storage}"
249
399
  end
250
400
 
251
401
  Yomikomu.info{ "[RUBY_YOMIKOMU] use #{STORAGE.class}" }
@@ -1,3 +1,3 @@
1
1
  module Yomikomu
2
- VERSION = "0.3.0"
2
+ VERSION = "0.4.0"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: yomikomu
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Koichi Sasada
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2016-04-12 00:00:00.000000000 Z
11
+ date: 2016-05-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: mmapped_string
@@ -120,7 +120,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
120
120
  version: '0'
121
121
  requirements: []
122
122
  rubyforge_project:
123
- rubygems_version: 2.6.3
123
+ rubygems_version: 2.6.4
124
124
  signing_key:
125
125
  specification_version: 4
126
126
  summary: Dump compiled iseq by binary (kakidasu) and load binary (yomidasu).