yomikomu 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 547fca221d04dc83b57b2aa5dfe64d02c0707a85
-  data.tar.gz: 3b69bf15676c15ddbc0aacd588ab34817e7a63c8
+  metadata.gz: 6e1bcc1dbf78cadc55edd354c6a420358558e084
+  data.tar.gz: da90ed8f437df5aadbbb6b10b73e9f783b51b2aa
 SHA512:
-  metadata.gz: 822c25e871fefe58dcdcbed67739fcf9157d59e8011981bff9c3ea3b51f3c47d5fe7c01db9397076035716038c7ffeb96640e5b41c69fea443b26a4669a66883
-  data.tar.gz: 86db6de8c2ac86e679fde45a4ef1de8f4f48bc406adf55e7fe64a568d5cca0c6e128d38f7d05f49651d1dc72832eea43cb9db28dccd4f9e903eb24f46e158413
+  metadata.gz: 4a4b925c4c284dc225ad5d92905685d7ee48d5daad135acf05d5c2a89949023ae43fb52dbc50f3574ccda99e53eebd8ca73d7ad6b3c3c81bf56944c7b447a555
+  data.tar.gz: 0302472fcbb4dd4f0e78d040998ce0302df08e5debaef057cd9a5f8bd17c9e1adb12433c5f93fb4cc4bc29efb36aaa0a769253c132e844c40ee79266403aebda

data/README.md

@@ -31,9 +31,13 @@ You can use the following configuration with environment variables.
 * YOMIKOMU_STORAGE (default: fs): choose storage type.
   * 'fs' (default) stores binaries in the same directory (for examlple, `x.rb` will be compiled to `x.rb.yarb` in the same directory).
   * 'fs2' stores binaries in specific directory.
+  * 'fsgz' stores files same as 'fs', but gz compressed.
+  * 'fs2gz' stores files same as 'fs2', but gz compressed.
   * 'dbm' stores binaries using dbm
+  * 'flatfile' stores binaries in one flat file. Updating is not supported. Use with YOMIKOMU_AUTO_COMPILE.
 * YOMIKOMU_STORAGE_DIR (default: "~/.ruby_binaries"): choose directory where binary files are stored.
 * YOMIKOMU_AUTO_COMPILE (default: false): if this value is `true`, then compile all required scripts implicitly.
+* YOMIKOMU_INFO (default: not defaind): show some information.
 * YOMIKOMU_DEBUG (default: not defined): show many information.
 
 ### Compile and store instruction sequences

data/exe/kakidasu

@@ -2,6 +2,8 @@
 
 $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 
+require 'yomikomu'
+require 'rbconfig'
 require 'optparse'
 
 verbose = false
@@ -37,8 +39,6 @@ executor = Proc.new{|file|
 
 opts.parse!(ARGV)
 
-require 'yomikomu'
-require 'rbconfig'
 paths = ARGV.empty? ? [RbConfig::CONFIG['libdir']] : ARGV
 
 paths.each{|path|

data/lib/yomikomu.rb

@@ -3,22 +3,23 @@ require "yomikomu/version"
 module Yomikomu
   STATISTICS = Hash.new(0)
 
-  unless yomu_dir = ENV['YOMIKOMU_STORAGE_DIR']
-    yomu_dir = File.expand_path("~/.ruby_binaries")
-    unless File.exist?(yomu_dir)
-      Dir.mkdir(yomu_dir)
+  def self.prefix
+    unless yomu_dir = ENV['YOMIKOMU_STORAGE_DIR']
+      yomu_dir = File.expand_path("~/.ruby_binaries")
     end
+    Dir.mkdir(yomu_dir) unless File.exist?(yomu_dir)
+    "#{yomu_dir}/cb."
   end
-  YOMIKOMU_PREFIX = "#{yomu_dir}/cb."
+
   YOMIKOMU_AUTO_COMPILE = ENV['YOMIKOMU_AUTO_COMPILE'] == 'true'
   YOMIKOMU_USE_MMAP = ENV['YOMIKOMU_USE_MMAP']
 
   def self.status
     STDERR.puts "[YOMIKOMU:INFO] (pid:#{Process.pid}) " +
-                ::Yomikomu::STATISTICS.map{|k, v| "#{k}: #{v}"}.join(' ,')
+                ::Yomikomu::STATISTICS.map{|k, v| "#{k}: #{v}"}.join(', ')
   end
 
-  if $VERBOSE
+  if $VERBOSE || ENV['YOMIKOMU_INFO'] == 'true'
     def self.info
       STDERR.puts "[YOMIKOMU:INFO] (pid:#{Process.pid}) #{yield}"
     end
@@ -74,14 +75,18 @@ module Yomikomu
       "SHA-1:#{::Digest::SHA1.file(fname).digest}"
     end
 
-    def compile_and_store_iseq fname, iseq_key = iseq_key_name(fname)
-      ::Yomikomu::STATISTICS[:compiled] += 1
+    def compile_and_store_iseq fname, iseq_key = iseq_key_name(fname = File.expand_path(fname))
       ::Yomikomu.debug{ "compile #{fname} into #{iseq_key}" }
-      iseq = RubyVM::InstructionSequence.compile_file(fname)
-
-      binary = iseq.to_binary(extra_data(fname))
-      write_compiled_iseq(fname, iseq_key, binary)
-      iseq
+      begin
+        iseq = RubyVM::InstructionSequence.compile_file(fname)
+        binary = iseq.to_binary(extra_data(fname))
+        write_compiled_iseq(fname, iseq_key, binary)
+        ::Yomikomu::STATISTICS[:compiled] += 1
+        iseq
+      rescue SyntaxError, RuntimeError => e
+        puts "#{e}: #{fname}"
+        nil
+      end
     end
 
     # def remove_compiled_iseq fname; nil; end # should implement at sub classes
@@ -102,11 +107,6 @@ module Yomikomu
   class FSStorage < BasicStorage
     def initialize
       super
-      require 'fileutils'
-      @dir = YOMIKOMU_PREFIX + "files"
-      unless File.directory?(@dir)
-        FileUtils.mkdir_p(@dir)
-      end
     end
 
     def remove_compiled_iseq fname
@@ -144,7 +144,21 @@ module Yomikomu
     end
   end
 
+  class FSSGZtorage < FSStorage
+
+  end
+
   class FS2Storage < FSStorage
+    def initialize
+      super
+
+      require 'fileutils'
+      @dir = Yomikomu.prefix + "files"
+      unless File.directory?(@dir)
+        FileUtils.mkdir_p(@dir)
+      end
+    end
+
     def iseq_key_name fname
       File.join(@dir, fname.gsub(/[^A-Za-z0-9\._-]/){|c| '%02x' % c.ord} + '.yarb') # special directory
     end
@@ -157,6 +171,37 @@ module Yomikomu
     end
   end
 
+  module GZFileStorage
+    def initialize
+      require 'zlib'
+      super
+    end
+
+    def iseq_key_name fname
+      super + '.gz'
+    end
+
+    def read_compiled_iseq fname, iseq_key
+      Zlib::GzipReader.open(iseq_key){|f|
+        f.read
+      }
+    end
+
+    def write_compiled_iseq fname, iseq_key, binary
+      Zlib::GzipWriter.open(iseq_key){|f|
+        f.write(binary)
+      }
+    end
+  end
+
+  class FSGZStorage < FSStorage
+    include GZFileStorage
+  end
+
+  class FS2GZStorage < FS2Storage
+    include GZFileStorage
+  end
+
   if YOMIKOMU_USE_MMAP
     require 'mmapped_string'
     Yomikomu.info{ "[RUBY_YOMIKOMU] use mmap" }
@@ -178,8 +223,9 @@ module Yomikomu
 
   class DBMStorage < BasicStorage
     def initialize
+      super
       require 'dbm'
-      @db = DBM.open(YOMIKOMU_PREFIX+'db')
+      @db = DBM.open(Yomikomu.prefix + 'db')
     end
 
     def remove_compiled_iseq fname
@@ -218,6 +264,100 @@ module Yomikomu
     end
   end
 
+  class FlatFileStorage < BasicStorage
+    def index_path
+      Yomikomu.prefix + 'ff_index'
+    end
+
+    def data_path
+      Yomikomu.prefix + 'ff_data'
+    end
+
+    def initialize
+      super
+      require 'fileutils'
+
+      @updated = false
+
+      if File.exist?(index_path)
+        open(index_path, 'rb'){|f| @index = Marshal.load(f)}
+      else
+        @index = {}
+        open(data_path, 'w'){} # touch
+      end
+
+      @data_file = open(data_path, 'a+b')
+
+      at_exit{
+        if @updated
+          open(index_path, 'wb'){|f| Marshal.dump(@index, f)}
+          Yomikomu.info{'FlatFile: update'}
+        end
+      }
+    end
+
+    def remove_compiled_iseq fname
+      raise 'unsupported'
+    end
+
+    private
+
+    def compiled_iseq_exist? fname, iseq_key
+      @index[iseq_key]
+    end
+
+    def compiled_iseq_is_younger? fname, iseq_key
+      offset, size, date = @index[iseq_key]
+      date.to_i >= File.mtime(fname).to_i
+    end
+
+    def read_compiled_iseq fname, iseq_key
+      offset, size, date = @index[iseq_key]
+      @data_file.pos = offset
+      binary = @data_file.read(size)
+      raise "size is not match" if binary.size != size
+      binary
+    end
+
+    def write_compiled_iseq fname, iseq_key, binary
+      raise "compiled binary for #{fname} already exists. flatfile does not support overwrite." if compiled_iseq_exist?(fname, iseq_key)
+
+      @data_file.seek 0, IO::SEEK_END
+      offset = @data_file.tell
+      size = binary.size
+      date = Time.now.to_i
+      @data_file.write(binary)
+      @index[iseq_key] = [offset, size, date]
+
+      @updated = true
+    end
+  end
+
+  class FlatFileGZStorage < FlatFileStorage
+    def index_path
+      super + '_gz'
+    end
+
+    def data_path
+      super + '_gz'
+    end
+
+    def initialize
+      super
+      require 'zlib'
+    end
+
+    def read_compiled_iseq fname, iseq_key
+      binary = super
+      Zlib::Inflate.inflate(binary)
+    end
+
+    def write_compiled_iseq fname, iseq_key, binary
+      binary = Zlib::Deflate.deflate(binary)
+      super(fname, iseq_key, binary)
+    end
+  end
+
   def self.compile_and_store_iseq fname
     STORAGE.compile_and_store_iseq fname
   end
@@ -235,17 +375,27 @@ module Yomikomu
   end
 
   # select storage
-  STORAGE = case ENV['YOMIKOMU_STORAGE']
-            when 'dbm'
-              DBMStorage.new
+  STORAGE = case storage = ENV['YOMIKOMU_STORAGE']
             when 'fs'
               FSStorage.new
+            when 'fsgz'
+              FSGZStorage.new
             when 'fs2'
               FS2Storage.new
+            when 'fs2gz'
+              FS2GZStorage.new
+            when 'dbm'
+              DBMStorage.new
+            when 'flatfile'
+              FlatFileStorage.new
+            when 'flatfilegz'
+              FlatFileGZStorage.new
             when 'null'
               NullStorage.new
-            else
+            when nil
               FSStorage.new
+            else
+              raise "Unknown storage type: #{storage}"
             end
 
   Yomikomu.info{ "[RUBY_YOMIKOMU] use #{STORAGE.class}" }

data/lib/yomikomu/version.rb

@@ -1,3 +1,3 @@
 module Yomikomu
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: yomikomu
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Koichi Sasada
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-04-12 00:00:00.000000000 Z
+date: 2016-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mmapped_string
@@ -120,7 +120,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.3
+rubygems_version: 2.6.4
 signing_key: 
 specification_version: 4
 summary: Dump compiled iseq by binary (kakidasu) and load binary (yomidasu).