yomikomu 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c79155668190a8c88087c6c5742860d5a67d838a
+  data.tar.gz: 48d6334da030db6727f082884982fb3046d7d4e7
+SHA512:
+  metadata.gz: 2e630562b6fcc1e40d51e8dd22aae77b7048ffe64a3433a229fe8a0ae8c85be30cc6a1af4c0f2780d00fb74ebf5f2689d3cf06b0f335d064d408e9a06793bbd7
+  data.tar.gz: 03f1ae8cc3d2dc1e5b4c181db83c76d29d1d39cea9e3cbec3ec9a94d07e65a25a98c1f07783fc027415a6860004321aa2d280e29c60fccb1e1a2ea3147283569

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.3.0
+before_install: gem install bundler -v 1.10.6

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in yomikomu.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Koichi Sasada
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,68 @@
+# Yomikomu
+
+* `yomikomu` gem enables to load compiled instruction sequences (bytecodes).
+* `kakidasu` command compiles and stores compiled instruction sequences (bytecodes).
+
+This gem requires Ruby 2.3.0.
+Most of code is ported from `ruby/sample/iseq_load.rb`.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'yomikomu'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install yomikomu
+
+## Usage
+
+### Configuration
+
+You can use the following configuration with environment variables.
+
+* YOMIKOMU_STORAGE (default: fs): choose storage type.
+  * 'fs' (default) stores binaries in the same directory (for examlple, `x.rb` will be compiled to `x.rb.yarb` in the same directory).
+  * 'fs2' stores binaries in specific directory.
+  * 'dbm' stores binaries using dbm
+* YOMIKOMU_STORAGE_DIR (default: "~/.ruby_binaries"): choose directory where binary files are stored.
+* YOMIKOMU_AUTO_COMPILE (default: false): if this value is `true`, then compile all required scripts implicitly.
+* YOMIKOMU_DEBUG (default: not defined): show many information.
+
+### Compile and store instruction sequences
+
+You only need to use the following command.
+
+```
+$ kakidasu [file or dir] ...
+```
+
+Compile specified file or files (*.rb) in specified directory.
+If no files or directories are specified, then use "libdir" of installed Ruby.
+
+### Load compiled binary
+
+You only need to require `yomikomu`.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/ko1/yomikomu.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "yomikomu"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/kakidasu

@@ -0,0 +1,53 @@
+#! /usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+
+require 'optparse'
+
+verbose = false
+executor = nil
+
+opts = OptionParser.new{|o|
+  o.on('--verify [original script]', 'verify mode'){
+    executor = Proc.new{|file|
+      raise "TODO"
+    }
+  }
+  o.on('--remove [original script]', 'remove specific compiled iseq'){
+    executor = Proc.new{|file|
+      ::Yomikomu.remove_compiled_iseq(file)
+    }
+  }
+  o.on('--remove-all', 'remove all compiled iseq'){
+    ::Yomikomu.remove_all_compiled_iseq
+  }
+  o.on('-v', '--verbose'){
+    $VERBOSE = true
+    ENV['YOMIKOMU_DEBUG'] = 'true'
+  }
+}
+
+executor = Proc.new{|file|
+  begin
+    ::Yomikomu.compile_and_store_iseq(file)
+  rescue SyntaxError => e
+    STDERR.puts e
+  end
+} unless executor # default
+
+opts.parse!(ARGV)
+
+require 'yomikomu'
+require 'rbconfig'
+paths = ARGV.empty? ? [RbConfig::CONFIG['libdir']] : ARGV
+
+paths.each{|path|
+  if File.directory?(path)
+    pattern = File.join(path, '**/*.rb')
+    Dir.glob(pattern){|file|
+      executor.call(file)
+    }
+  else
+    executor.call(path)
+  end
+}

data/lib/yomikomu/version.rb

@@ -0,0 +1,3 @@
+module Yomikomu
+  VERSION = "0.1.0"
+end

data/lib/yomikomu.rb

@@ -0,0 +1,234 @@
+require "yomikomu/version"
+
+module Yomikomu
+  class NullStorage
+    def load_iseq fname; end
+    def compile_and_store_iseq fname; end
+    def remove_compiled_iseq fname; end
+  end
+
+  STATISTICS = Hash.new(0)
+
+  unless yomu_dir = ENV['YOMIKOMU_STORAGE_DIR']
+    yomu_dir = File.expand_path("~/.ruby_binaries")
+    unless File.exist?(yomu_dir)
+      Dir.mkdir(yomu_dir)
+    end
+  end
+  YOMIKOMU_PREFIX = "#{yomu_dir}/cb."
+  YOMIKOMU_AUTO_COMPILE = ENV['YOMIKOMU_AUTO_COMPILE'] == 'true'
+
+  if $VERBOSE
+    def self.info
+      STDERR.puts "[YOMIKOMU:INFO] (pid:#{Process.pid}) #{yield}"
+    end
+    at_exit{
+      STDERR.puts "[YOMIKOMU:INFO] (pid:#{Process.pid}) " +
+                  ::Yomikomu::STATISTICS.map{|k, v| "#{k}: #{v}"}.join(' ,')
+    }
+  else
+    def self.info
+    end
+  end
+
+  if ENV['YOMIKOMU_DEBUG'] == 'true'
+    def self.debug
+      STDERR.puts "[YOMIKOMU:DEBUG] (pid:#{Process.pid}) #{yield}"
+    end
+  else
+    def self.debug
+    end
+  end
+
+  class BasicStorage
+    def initialize
+      require 'digest/sha1'
+    end
+
+    def load_iseq fname
+      iseq_key = iseq_key_name(fname)
+
+      if compiled_iseq_exist?(fname, iseq_key) && compiled_iseq_is_younger?(fname, iseq_key)
+        ::Yomikomu::STATISTICS[:loaded] += 1
+        ::Yomikomu.debug{ "load #{fname} from #{iseq_key}" }
+        binary = read_compiled_iseq(fname, iseq_key)
+        iseq = RubyVM::InstructionSequence.load_from_binary(binary)
+        # p [extra_data(iseq.path), RubyVM::InstructionSequence.load_from_binary_extra_data(binary)]
+        # raise unless extra_data(iseq.path) == RubyVM::InstructionSequence.load_from_binary_extra_data(binary)
+        iseq
+      elsif YOMIKOMU_AUTO_COMPILE
+        compile_and_store_iseq(fname, iseq_key)
+      else
+        ::Yomikomu::STATISTICS[:ignored] += 1
+        ::Yomikomu.debug{ "ignored #{fname}" }
+        nil
+      end
+    end
+
+    def extra_data fname
+      "SHA-1:#{::Digest::SHA1.file(fname).digest}"
+    end
+
+    def compile_and_store_iseq fname, iseq_key = iseq_key_name(fname)
+      ::Yomikomu::STATISTICS[:compiled] += 1
+      ::Yomikomu.debug{ "[RUBY_COMPILED_FILE] compile #{fname} into #{iseq_key}" }
+      iseq = RubyVM::InstructionSequence.compile_file(fname)
+
+      binary = iseq.to_binary(extra_data(fname))
+      write_compiled_iseq(fname, iseq_key, binary)
+      iseq
+    end
+
+    # def remove_compiled_iseq fname; nil; end # should implement at sub classes
+
+    private
+
+    def iseq_key_name fname
+      fname
+    end
+
+    # should implement at sub classes
+    # def compiled_iseq_younger? fname, iseq_key; end
+    # def compiled_iseq_exist? fname, iseq_key; end
+    # def read_compiled_file fname, iseq_key; end
+    # def write_compiled_file fname, iseq_key, binary; end
+  end
+
+  class FSStorage < BasicStorage
+    def initialize
+      super
+      require 'fileutils'
+      @dir = YOMIKOMU_PREFIX + "files"
+      unless File.directory?(@dir)
+        FileUtils.mkdir_p(@dir)
+      end
+    end
+
+    def remove_compiled_iseq fname
+      iseq_key = iseq_key_name(fname)
+      if File.exist?(iseq_key)
+        Yomikomu.debug{ "rm #{iseq_key}" }
+        File.unlink(iseq_key)
+      end
+    end
+
+    private
+
+    def iseq_key_name fname
+      "#{fname}.yarb" # same directory
+    end
+
+    def compiled_iseq_exist? fname, iseq_key
+      File.exist?(iseq_key)
+    end
+
+    def compiled_iseq_is_younger? fname, iseq_key
+      File.mtime(iseq_key) >= File.mtime(fname)
+    end
+
+    def read_compiled_iseq fname, iseq_key
+      File.binread(iseq_key)
+    end
+
+    def write_compiled_iseq fname, iseq_key, binary
+      File.binwrite(iseq_key, binary)
+    end
+
+    def remove_all_compiled_iseq
+      raise "unsupported"
+    end
+  end
+
+  class FS2Storage < FSStorage
+    def iseq_key_name fname
+      File.join(@dir, fname.gsub(/[^A-Za-z0-9\._-]/){|c| '%02x' % c.ord} + '.yarb') # special directory
+    end
+
+    def remove_all_compiled_iseq
+      Dir.glob(File.join(@dir, '**/*.yarb')){|path|
+        Yomikomu.debug{ "rm #{path}" }
+        FileUtils.rm(path)
+      }
+    end
+  end
+
+  class DBMStorage < BasicStorage
+    def initialize
+      require 'dbm'
+      @db = DBM.open(YOMIKOMU_PREFIX+'db')
+    end
+
+    def remove_compiled_iseq fname
+      @db.delete fname
+    end
+
+    private
+
+    def date_key_name fname
+      "date.#{fname}"
+    end
+
+    def iseq_key_name fname
+      "body.#{fname}"
+    end
+
+    def compiled_iseq_exist? fname, iseq_key
+      @db.has_key? iseq_key
+    end
+
+    def compiled_iseq_is_younger? fname, iseq_key
+      date_key = date_key_name(fname)
+      if @db.has_key? date_key
+        @db[date_key].to_i >= File.mtime(fname).to_i
+      end
+    end
+
+    def read_compiled_iseq fname, iseq_key
+      @db[iseq_key]
+    end
+
+    def write_compiled_iseq fname, iseq_key, binary
+      date_key = date_key_name(fname)
+      @db[iseq_key] = binary
+      @db[date_key] = Time.now.to_i
+    end
+  end
+
+  def self.compile_and_store_iseq fname
+    STORAGE.compile_and_store_iseq fname
+  end
+
+  def self.remove_compiled_iseq fname
+    STORAGE.remove_compiled_iseq fname
+  end
+
+  def self.remove_all_compiled_iseq
+    STORAGE.remove_all_compiled_iseq
+  end
+
+  def self.verify_compiled_iseq fname
+    STORAGE.verify_compiled_iseq fname
+  end
+
+  # select storage
+  STORAGE = case ENV['YOMIKOMU_STORAGE']
+            when 'dbm'
+              DBMStorage.new
+            when 'fs'
+              FSStorage.new
+            when 'fs2'
+              FS2Storage.new
+            when 'null'
+              NullStorage.new
+            else
+              FSStorage.new
+            end
+
+  Yomikomu.info{ "[RUBY_YOMIKOMU] use #{STORAGE.class}" }
+end
+
+class RubyVM::InstructionSequence
+  def self.load_iseq fname
+    ::Yomikomu::STORAGE.load_iseq(fname)
+  end
+end

data/yomikomu.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'yomikomu/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "yomikomu"
+  spec.version       = Yomikomu::VERSION
+  spec.authors       = ["Koichi Sasada"]
+  spec.email         = ["ko1@atdot.net"]
+
+  spec.summary       = %q{Dump compiled iseq by binary (kakidasu) and load binary (yomidasu).}
+  spec.description   = %q{Dump compiled iseq by binary (kakidasu) and load binary (yomidasu).}
+  spec.homepage      = "http://github.com/ko1/yomikomu"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest"
+end

metadata

@@ -0,0 +1,99 @@
+--- !ruby/object:Gem::Specification
+name: yomikomu
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Koichi Sasada
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-12-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Dump compiled iseq by binary (kakidasu) and load binary (yomidasu).
+email:
+- ko1@atdot.net
+executables:
+- kakidasu
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/kakidasu
+- lib/yomikomu.rb
+- lib/yomikomu/version.rb
+- yomikomu.gemspec
+homepage: http://github.com/ko1/yomikomu
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.0
+signing_key: 
+specification_version: 4
+summary: Dump compiled iseq by binary (kakidasu) and load binary (yomidasu).
+test_files: []