script_core 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bfb29eda17a289a639eaeda3bd447732d774ea5c9a6f7ec5e3d71406891d3376
-  data.tar.gz: 4245f1728f35920db4d370805ad38951f9ae9695807a7a090ba799207f8a6acf
+  metadata.gz: 130d5ba4a71897c28741286895a90e289929e0f4e46e55cb7a80a95f00636d90
+  data.tar.gz: 93aae35e4585217c81940534e52c3aff4bacc73ce0df9b193a5ef77512cc60f2
 SHA512:
-  metadata.gz: c9d7e1a4acfe78e4a05371e44955bc6032b4844384eacac24a188bd3f9a2737be592f0ee39845d16bec1ee18244a63d5627492275593a2953844f57f6b9fbe7f
-  data.tar.gz: 463bfd37532feea17dbad1c91e3cd5c35aecb93162c1fe40209198f7fdd4d43b4fe5f899d24a37f79fd39ff75afba5969a3eb90c04ee247b60d2b7671cba2564
+  metadata.gz: 1f2d15a9c3ea873905c39456cf27ea15a7878e7ef3e7420e70e9da98e67ff05bd0bb1ccca29b0811bafb007e8d2715a861f41ffe01ad78e4e77cb7c812847454
+  data.tar.gz: 4bedfecfd8d286779cbb03788ab3710fee025ace61d57300248d0adb0a7ce85d362134594f2a3d424c6852ed2fac7ca2779db67934605d0ff36c47a5ca156224

data/.gitattributes

@@ -1,3 +1,3 @@
 *.rb diff=ruby
+*.rake diff=ruby
 *.gemspec diff=ruby
-

data/.gitignore

@@ -2,6 +2,7 @@
 .vagrant
 .bundle
 *.gem
+*.mrb
 Gemfile.lock
 tmp
 bin/enterprise_script_service
@@ -9,5 +10,18 @@ bin/enterprise_script_service_tests
 bin/enterprise_script_service.dSYM
 bin/enterprise_script_service_tests.dSYM
 bin/mrbc
-bin/mruby
 spec/examples.txt
+
+spec/dummy/db/*.sqlite3
+spec/dummy/db/*.sqlite3-journal
+spec/dummy/log/*.log
+spec/dummy/node_modules/
+spec/dummy/yarn-error.log
+spec/dummy/tmp/
+
+# Ignore uploaded files in development
+/spec/dummy/storage/*
+!/spec/dummy/storage/.keep
+
+# Ignore compiled mruby in dummy app
+/spec/dummy/mruby/bin

data/.rubocop.yml

@@ -1,5 +1,5 @@
 AllCops:
-  TargetRubyVersion: 2.3
+  TargetRubyVersion: 2.4
   Exclude:
     - bin/**/*
     - tests/**/*
@@ -7,6 +7,8 @@ AllCops:
     - ext/enterprise_script_service/mruby-mpdecimal/**/*
     - ext/enterprise_script_service/msgpack/**/*
     - ext/enterprise_script_service/libseccomp/**/*
+    - spec/dummy/mruby/**/*
+    - spec/dummy/db/schema.rb
 
 Metrics/LineLength:
   Max: 150
@@ -123,7 +125,7 @@ Style/UnneededPercentQ:
 
 # Align `end` with the matching keyword or starting expression except for
 # assignments, where it should be aligned with the LHS.
-Lint/EndAlignment:
+Layout/EndAlignment:
   Enabled: true
   EnforcedStyleAlignWith: variable
 
@@ -154,3 +156,9 @@ Metrics/ClassLength:
 
 Layout/EmptyLinesAroundArguments:
   Enabled: false
+
+Style/ClassAndModuleChildren:
+  Enabled: false
+
+Naming/UncommunicativeMethodParamName:
+  Enabled: false

data/.ruby-version

@@ -1 +1 @@
-2.5.0
+2.5.3

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+
+rvm:
+  - 2.5.1
+
+script: bundle exec rake compile

data/Gemfile

@@ -4,3 +4,35 @@ source("https://rubygems.org")
 git_source(:github) { |repo| "https://github.com/#{repo}.git" }
 
 gemspec
+
+# Dummy app
+gem "rails", "~> 5.2"
+gem "sqlite3"
+
+# Use Puma as the app server
+gem "puma"
+# For better console experience
+gem "pry-rails"
+
+gem "listen", ">= 3.0.5", "< 3.2"
+# Access an IRB console on exception pages or by using <%= console %> anywhere in the code.
+gem "web-console"
+# Call "byebug" anywhere in the code to stop execution and get a debugger console
+gem "pry-byebug"
+
+# To support ES6
+gem "sprockets", "~> 4.0.0.beta4"
+# Support ES6
+gem "babel-transpiler"
+# Use SCSS for stylesheets
+gem "sassc-rails"
+# Use Uglifier as compressor for JavaScript assets
+gem "uglifier", ">= 1.3.0"
+
+gem "cocoon"
+gem "form_core"
+
+gem "bulma-rails"
+gem "jquery-rails"
+gem "selectize-rails"
+gem "turbolinks", "~> 5"

data/README.md

@@ -1,12 +1,175 @@
 ScriptCore
 ====
 
-ScriptCore is forked by Shopify's enterprise script service.
+> ScriptCore is under development, the codebase is unoptimized and has many bad practices, I may do breaking changes even force pushing to master branch.
+
+ScriptCore is a fork of Shopify's enterprise script service.
 
 The enterprise script service (aka ESS) is a thin Ruby API layer that spawns a process, the `enterprise_script_engine`, to execute an untrusted Ruby script.
 
 The `enterprise_script_engine` executable ingests the input from `stdin` as a msgpack encoded payload; then spawns an mruby-engine; uses seccomp to sandbox itself; feeds `library`, `input` and finally the Ruby scripts into the engine; returns the output as a msgpack encoded payload to `stdout` and finally exits.
 
+## Why fork?
+
+I want to make these changes:
+
+- Toolchain
+    - [x] Expose mruby build config to allow developer can modify mruby-engine executable, e.g: add some gems
+    - [x] Expose `mrbc` to allow developer precompile mruby library that would inject to sandboxie
+    - [ ] Watching and auto compiling mruby library when it change
+    - [x] Rake tasks for compiling mruby-engine & mruby library
+- Practice
+    - [x] Rails generator for mruby library
+    - [x] Find a good place for engines
+    - [ ] Find a good way for working with timezone in mruby
+    - [ ] Find a good way for working with `BigDecimal` & `Date` (mruby doesn't have these)
+    - [ ] Find a good way for secure IO
+
+## Demo
+
+Clone the repository.
+
+```sh
+$ git clone https://github.com/rails-engine/script_core
+```
+
+Change directory
+
+```sh
+$ cd script_core
+```
+
+Run bundler
+
+```sh
+$ bundle install
+```
+
+Preparing database
+
+```sh
+$ bin/rails db:migrate
+```
+
+Build mruby engine & engine lib
+
+```sh
+$ bin/rails app:script_core:engine:build
+$ bin/rails app:script_core:engine:compile_lib 
+```
+
+Start the Rails server
+
+```sh
+$ bin/rails s
+```
+
+Open your browser, and visit `http://localhost:3000`
+
+## Installation
+
+Add this line to your Gemfile:
+
+```ruby
+gem 'script_core'
+```
+
+Or you may want to include the gem directly from GitHub:
+
+```ruby
+gem 'script_core', github: 'rails-engine/script_core'
+```
+
+Then execute:
+
+```sh
+$ bundle
+```
+
+## Build your executable
+
+ScriptCore already has a default executable, because of mruby's gem is compiled in binary, or you may want to build a mruby library, build your own engine is necessary.
+
+You can check `spec/dummy/mruby` as reference.
+
+### Create a new engine
+
+Run the task in your app directory:
+
+```sh
+$ rails script_core:engine:new [engine_name]
+```
+
+`engine_name` is optional, by default it would be `mruby` that will generate `mruby` directory in your app root folder.
+
+Then execute:
+
+```sh
+$ rails script_core:engine:build [engine_name]
+```
+
+It will build mruby executables.
+
+#### customizing `gembox`
+
+Remove `.example` extension for `engine.gembox.example`, customize it, then rebuild the engine.
+
+**Warning: because of `seccomp`, you may meet compatibility problems, especially for IO relates gems.**
+
+### Build lib for the engine
+
+Write your own lib for mruby environment in `mruby/lib` directory.
+
+### Compile lib for the engine
+
+Run the task in your app directory:
+
+```sh
+$ rails script_core:engine:compile_lib [engine_name]
+```
+
+### Integrate to your app
+
+You can wrap it for example:
+
+```ruby
+module ScriptEngine
+  class << self
+    def engine
+      @engine ||= ScriptCore::Engine.new Rails.root.join("mruby/bin")
+    end
+
+    def eval(string, input: nil, instruction_quota_start: nil, environment_variables: {})
+      sources = [
+        ["user", string],
+      ]
+
+      engine.eval sources, input: input,
+                  instruction_quota_start: instruction_quota_start,
+                  environment_variables: environment_variables
+    end
+  end
+end
+```
+
+Then use it:
+
+```ruby
+ScriptEngine.eval "@output = 'hello world'"
+```
+
+## Tips
+
+- Add `/mruby/bin` into `.gitignore`
+- Don't do any IO in mruby side
+- Because of `seccomp`, it may have compatible issues with some mruby gems
+- mruby doesn't have `Date`, use `Time` instead
+- mruby doesn't have `BigDecimal`, you can use Shopify's `Decimal` instead
+- mruby is poor support timezone, you'd better handle it by yourself
+- mruby engine is fast, usually it only costs 3 - 5ms depends on complexity, but it consume a lot of memory (~300k at least per process)
+
+# More information about ESS
+
 ## Data format
 
 ### Input
@@ -102,11 +265,11 @@ Ruby code is in `lib/`
 ```
 $ vagrant up
 $ vagrant ssh
-vagrant@vagrant-ubuntu-trusty-64:~$ cd /vagrant
-vagrant@vagrant-ubuntu-trusty-64:/vagrant$ bundle install
-vagrant@vagrant-ubuntu-trusty-64:/vagrant$ git submodule init
-vagrant@vagrant-ubuntu-trusty-64:/vagrant$ git submodule update
-vagrant@vagrant-ubuntu-trusty-64:/vagrant$ bin/rake
+vagrant@vagrant-ubuntu-bionic-64:~$ cd /vagrant
+vagrant@vagrant-ubuntu-bionic-64:/vagrant$ bundle install
+vagrant@vagrant-ubuntu-bionic-64:/vagrant$ git submodule init
+vagrant@vagrant-ubuntu-bionic-64:/vagrant$ git submodule update
+vagrant@vagrant-ubuntu-bionic-64:/vagrant$ bin/rake
 ```
 
 ## Contributing

data/Rakefile

@@ -1,5 +1,11 @@
 # frozen_string_literal: true
 
+begin
+  require "bundler/setup"
+rescue LoadError
+  puts "You must `gem install bundler` and `bundle install` to run rake tasks"
+end
+
 require "bundler/gem_tasks"
 require "rake/extensiontask"
 
@@ -9,6 +15,11 @@ begin
 rescue LoadError
 end
 
+APP_RAKEFILE = File.expand_path("spec/dummy/Rakefile", __dir__)
+load "rails/tasks/engine.rake"
+
+load "rails/tasks/statistics.rake"
+
 task(compile: []) do
   Dir.chdir("ext/enterprise_script_service") do
     sh("../../bin/rake")

data/Vagrantfile

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 Vagrant.configure(2) do |config|
-  config.vm.box = "ubuntu/trusty64"
+  config.vm.box = "ubuntu/bionic64"
   config.vm.provision(:shell, path: "bootstrap.sh")
   config.ssh.forward_agent = true
 end

data/bin/rails

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+# This command will automatically be run when you run "rails" with Rails gems
+# installed from the root of your application.
+
+ENGINE_ROOT = File.expand_path('..', __dir__)
+ENGINE_PATH = File.expand_path('../lib/script_core/railtie', __dir__)
+APP_PATH = File.expand_path('../spec/dummy/config/application', __dir__)
+
+# Set up gems listed in the Gemfile.
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
+require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])
+
+require "rails"
+# Pick the frameworks you want:
+require "active_model/railtie"
+# require "active_job/railtie"
+require "active_record/railtie"
+# require "active_storage/engine"
+require "action_controller/railtie"
+# require "action_mailer/railtie"
+require "action_view/railtie"
+# require "action_cable/engine"
+require "sprockets/railtie"
+require "rails/test_unit/railtie"
+require "rails/engine/commands"

data/bootstrap.sh

@@ -3,9 +3,8 @@
 apt-get update
 apt-get install -y build-essential zlib1g-dev libssl-dev libreadline6-dev libyaml-dev wget libffi-dev gcc g++ make bison libtool autoconf git
 cd `mktemp -d`
-curl -L "https://cache.ruby-lang.org/pub/ruby/2.5/ruby-2.5.0.tar.gz" | tar -xzf-
-cd ruby-2.5.0
+curl -L "https://cache.ruby-lang.org/pub/ruby/2.5/ruby-2.5.3.tar.gz" | tar -xzf-
+cd ruby-2.5.3
 ./configure --prefix=/usr/local
 make
 make install
-gem install bundler

data/dev.yml

@@ -1,20 +1,20 @@
-name: enterprise-script-service
+name: script_core
 
 up:
   - xcode_clt
   - ruby:
-      version: 2.5.0
+      version: 2.5.3
   - bundler
   - submodules
 
 commands:
   recompile:
     syntax: ""
-    desc: "Compiles native extention"
+    desc: "Compiles native extension"
     run: bin/rake clobber compile
   compile:
     syntax: ""
-    desc: "Compiles native extention"
+    desc: "Compiles native extension"
     run: bin/rake compile
   spec:
     syntax: ""

data/ext/enterprise_script_service/Rakefile

@@ -14,9 +14,12 @@ raise(<<-MESSAGE) unless Dir.exist?(MRUBY_DIR.join("src"))
     $ git submodule update
 
 MESSAGE
+
 ESS_ROOT = Pathname.new(__dir__).join("../..")
 ROOT =
-  if ENV["INSTALL_DIR"] && File.directory?(ENV["INSTALL_DIR"])
+  if ENV["TARGET_DIR"] && File.directory?(ENV["TARGET_DIR"])
+    Pathname.new(ENV["TARGET_DIR"])
+  elsif ENV["INSTALL_DIR"] && File.directory?(ENV["INSTALL_DIR"])
     Pathname.new(ENV["INSTALL_DIR"])
   else
     ESS_ROOT
@@ -34,7 +37,6 @@ end
 
 MRUBY_BIN_DIR = MRUBY_DIR.join("build/host/bin")
 MRBC_EXE = MRUBY_BIN_DIR.join("mrbc")
-MRUBY_EXE = MRUBY_BIN_DIR.join("mruby")
 
 MRUBY_LIB_DIR = MRUBY_DIR.join("build/sandbox/lib")
 MRUBY_LIB = MRUBY_LIB_DIR.join("libmruby.a")
@@ -44,7 +46,7 @@ LIBSECCOMP_LIB_DIR = LIBSECCOMP_DIR.join("src/.libs")
 
 CXX = ENV["CXX"] || "g++"
 
-LIBSECCOMP_CFLAGS = if RUBY_PLATFORM =~ /linux/
+LIBSECCOMP_CFLAGS = if RUBY_PLATFORM.match?(/linux/)
   [
     "-Ilibseccomp/include",
     "-L#{LIBSECCOMP_LIB_DIR}",
@@ -79,7 +81,6 @@ file(SERVICE_EXECUTABLE => [
   )
 
   sh("cp", MRBC_EXE.to_s, SERVICE_EXECUTABLE_DIR.to_s)
-  sh("cp", MRUBY_EXE.to_s, SERVICE_EXECUTABLE_DIR.to_s)
 end
 
 SERVICE_SOURCES_NO_MAIN = SERVICE_SOURCES.reject { |f| f == "ext.cpp" }