pione 0.2.1 → 0.2.2

data/lib/pione/util/misc.rb

@@ -8,8 +8,8 @@ module Pione
       def ignore_exception(&b)
         begin
           b.call
-        rescue Exception
-          # do nothing
+        rescue Exception => e
+          ErrorReport.warn("the error ignored", nil, e, __FILE__, __LINE__)
         end
       end
 

data/lib/pione/util/zip.rb

@@ -0,0 +1,60 @@
+module Pione
+  module Util
+    module Zip
+      class << self
+        # Create a zip archive of the location.
+        #
+        # @param src [DataLocation]
+        #   the source directory location
+        # @param archive [DataLocation]
+        #   the archive location
+        def compress(src, archive)
+          # src should be a directory
+          raise ArgumentError.new(src) unless src.directory?
+
+          # make local path
+          _src = src.local
+          _archive = Location[Temppath.create]
+
+          # compress
+          ::Zip::Archive.open(_archive.path.to_s, ::Zip::CREATE) do |ar|
+            _src.rel_entries(rec: true).each do |relpath|
+              relpath = relpath.to_s
+              location = _src + relpath
+              if location.directory?
+                ar.add_dir(relpath)
+              else
+                ar.add_file(relpath, location.path.to_s)
+              end
+            end
+          end
+
+          # upload archive
+          _archive.move(archive)
+        end
+
+        # Expand the archive into the destination directory.
+        #
+        # @param archive [DataLocation]
+        #    the archive location
+        # @param dest [DataLocation]
+        #    the destination directory location
+        def uncompress(archive, dest)
+          _archive = archive.local
+          _dest = dest.local
+          ::Zip::Archive.open(_archive.path.to_s) do |ar|
+            ar.each do |f|
+              path = _dest + f.name
+              if f.directory?
+                path.mkdir
+              else
+                f.read(8192) {|chunk| path.append(chunk)}
+              end
+            end
+          end
+          _dest.move(dest)
+        end
+      end
+    end
+  end
+end

data/lib/pione/version.rb

@@ -1,4 +1,4 @@
 module Pione
   # version of pione
-  VERSION = "0.2.1"
+  VERSION = "0.2.2"
 end

data/pione.gemspec

@@ -32,7 +32,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency "hamster", "~> 0.4.3"
   gem.add_dependency "naming", "~> 0.1.0"
   gem.add_dependency "forwardablex", "~> 0.1.4"
-  gem.add_dependency "temppath", "~> 0.1.1"
+  gem.add_dependency "temppath", "~> 0.2.0"
   gem.add_dependency "ruby-xes", "~> 0.1.0"
   gem.add_dependency "sys-uname", "~> 0.9.2"
   gem.add_dependency "simple-identity", "~> 0.1.1"
@@ -40,6 +40,10 @@ Gem::Specification.new do |gem|
   gem.add_dependency "sys-cpu", "~> 0.7.1"
   gem.add_dependency "structx", "~> 0.1.0"
   gem.add_dependency "syslog-logger", "~> 1.6.8"
+  gem.add_dependency "zipruby", "~> 0.3.6"
+  gem.add_dependency "retriable", "~> 1.3.3"
+  gem.add_dependency "childprocess", "~> 0.3.9"
+  gem.add_dependency "git", "~> 1.2.5"
 
   # for locations
   gem.add_dependency "dropbox-sdk", "~> 1.5.1"

data/test/agent/spec_rule-provider.rb

@@ -4,7 +4,7 @@ describe "Pione::Agent::RuleProvider" do
   before do
     @ts = create_tuple_space_server
     @agent = Agent[:rule_provider].new(@ts)
-    doc = Component::Document.load(<<-DOCUMENT)
+    doc = Component::Document.parse(<<-DOCUMENT)
       Rule A
         input  '*.a'
         output '{$I[1]}.result'
@@ -35,7 +35,7 @@ describe "Pione::Agent::RuleProvider" do
   end
 
   it "should add a rule" do
-    doc = Component::Document.load(<<-DOCUMENT)
+    doc = Component::Document.parse(<<-DOCUMENT)
       Rule C
         input  '*.c'
         output '{$I[1]}.result'

data/test/command/spec_pione-package.rb

@@ -0,0 +1,15 @@
+require_relative "../test-util"
+
+describe "Pione::Command::PionePackage" do
+  it "should build PIONE package" do
+    package_location = TestUtil::Package.get("TestPackage1")
+    output_location = Location[Temppath.mkdir]
+    res = TestUtil::Command.succeed do
+      Command::PionePackage.run ["--build", "-o", output_location.path.to_s, package_location.path.to_s]
+    end
+    pkg = output_location + "TestPackage1+v0.1.0.ppg"
+    pkg.should.exist
+    pkg.size.should > 0
+  end
+end
+

data/test/component/spec_document.rb

@@ -17,7 +17,7 @@ describe 'Pione::Component::Document' do
   end
 
   it 'should load a document from a string' do
-    Component::Document.load(document_location.read).should == @document
+    Component::Document.parse(document_location.read).should == @document
   end
 
   it 'should load a document with package name' do
@@ -58,7 +58,7 @@ describe 'Pione::Component::Document' do
 
   it 'should raise variable binding error' do
     should.raise(VariableBindingError) do
-      Component::Document.load <<-PIONE
+      Component::Document.parse <<-PIONE
         $X := 1
         $X := 2
       PIONE

data/test/component/spec_package-archiver.rb

@@ -0,0 +1,27 @@
+require_relative '../test-util'
+
+describe "Pione::Component::PackageArchiver" do
+  before do
+    @path = TestUtil::Package.get("TestPackage1")
+  end
+
+  it "should create archive file" do
+    out = Location[Temppath.mkdir]
+    Component::PackageArchiver.new(@path).archive(out)
+
+    # easy check
+    pkg = out + "TestPackage1+v0.1.0.ppg"
+    pkg.should.exist
+    pkg.should.file
+    pkg.size.should > 0
+
+    # structure check
+    Zip::Archive.open(pkg.path.to_s) do |ar|
+      ar.each do |file|
+        unless file.directory?
+          file.read.should == (@path + file.name).read
+        end
+      end
+    end
+  end
+end

data/test/component/spec_package-expander.rb

@@ -0,0 +1,11 @@
+require_relative '../test-util'
+
+describe "Pione::Component::PackageExpander" do
+  it "should expand archive package" do
+    location = TestUtil::TEST_PACKAGE_DIR + "TestPackage1+v0.1.0.ppg"
+    expanded = Location[Temppath.mkdir]
+    Component::PackageExpander.new(location).expand(expanded)
+    (expanded + "package.yml").should.exist
+  end
+end
+

data/test/component/spec_package-filename.rb

@@ -0,0 +1,79 @@
+require_relative '../test-util'
+
+shared "package filename" do
+  it "should parse the package filename" do
+    Component::PackageFilename.parse(@filename).tap do |fname|
+      fname.package_name.should == @package_name
+      fname.edition.should == @edition
+      fname.tag.should == @tag
+      fname.hash_id.should == @hash_id
+    end
+  end
+
+  it "should build package filename string" do
+    Component::PackageFilename.parse(@filename).to_s.should == @filename
+  end
+end
+
+describe "Pione::Component::PackageFilename" do
+  describe "package name + edition + tag + hash_id" do
+    before do
+      @filename = "Test(keita.yamaguchi@gmail.com)+test@d462c59.ppg"
+      @package_name = "Test"
+      @edition = "keita.yamaguchi@gmail.com"
+      @tag = "test"
+      @hash_id = "d462c59"
+    end
+
+    behaves_like "package filename"
+  end
+
+  describe "package name + 'origin' + tag + hash_id" do
+    before do
+      @filename = "Test+test@d462c59.ppg"
+      @package_name = "Test"
+      @edition = "origin"
+      @tag = "test"
+      @hash_id = "d462c59"
+    end
+
+    behaves_like "package filename"
+  end
+
+  describe "package name + tag + hash_id" do
+    before do
+      @filename = "Test+test@d462c59.ppg"
+      @package_name = "Test"
+      @edition = "origin"
+      @tag = "test"
+      @hash_id = "d462c59"
+    end
+
+    behaves_like "package filename"
+  end
+
+  describe "package name + tag" do
+    before do
+      @filename = "Test+test.ppg"
+      @package_name = "Test"
+      @edition = "origin"
+      @tag = "test"
+      @hash_id = nil
+    end
+
+    behaves_like "package filename"
+  end
+
+  describe "package name + hash id" do
+    before do
+      @filename = "Test@d462c59.ppg"
+      @package_name = "Test"
+      @edition = "origin"
+      @tag = nil
+      @hash_id = "d462c59"
+    end
+
+    behaves_like "package filename"
+  end
+end
+

data/test/component/spec_package-reader.rb

@@ -0,0 +1,100 @@
+require_relative '../test-util'
+
+$stdout = STDOUT
+$stderr = STDERR
+
+describe "Pione::Component::PackageReader" do
+  shared "package" do
+    it "should read the package and return it" do
+      Component::PackageReader.read(@location).should.kind_of Component::Package
+    end
+
+    it "should get package informations" do
+      package = Component::PackageReader.read(@location)
+      package.name.should == "HelloWorld"
+    end
+
+    it "should get scenarios" do
+      package = Component::PackageReader.read(@location)
+      package.name.should == "HelloWorld"
+      scenario = package.scenarios[0]
+      scenario.name.should == "HelloWorld"
+      scenario.inputs.should.empty
+      scenario.outputs[0].basename.should == "message.txt"
+
+      # case1 = package.scenarios[0]
+      # case1.name.should == "Case1"
+      # case1.inputs[0].basename.should == "1.txt"
+      # case2 = package.scenarios[1]
+      # case2.name.should == "Case2"
+      # case2.inputs[0].basename.should == "1.txt"
+      # case2.inputs[1].basename.should == "2.txt"
+      # case2.inputs[2].basename.should == "3.txt"
+      # case3 = package.scenarios[2]
+      # case3.name.should == "Case3"
+      # case3.inputs[0].basename.should == "a.txt"
+      # case3.inputs[1].basename.should == "b.txt"
+    end
+  end
+
+  shared "directory package" do
+    it "should read package directory" do
+      Component::PackageReader.new(@location).type.should == :directory
+    end
+  end
+
+  describe "directory package at local location" do
+    before do
+      @location = TestUtil::Package.get("HelloWorld")
+    end
+
+    behaves_like "package"
+    behaves_like "directory package"
+  end
+
+  describe "directory package at HTTP location" do
+    before do
+      @server = TestUtil::WebServer.start(TestUtil::TEST_PACKAGE_DIR)
+      @location = @server.root + "HelloWorld/"
+    end
+
+    after do
+      @server.terminate
+    end
+
+    behaves_like "package"
+    behaves_like "directory package"
+  end
+
+  describe "archive package at local location" do
+    before do
+      @location = TestUtil::TEST_PACKAGE_DIR + "HelloWorld+v0.1.0.ppg"
+    end
+
+    behaves_like "package"
+  end
+
+  describe "archive package at HTTP location" do
+    before do
+      @server = TestUtil::WebServer.start(TestUtil::TEST_PACKAGE_DIR)
+      @location = @server.root + "HelloWorld+v0.1.0.ppg"
+    end
+
+    after do
+      @server.terminate
+    end
+
+    behaves_like "package"
+  end
+
+  describe "git package at local location" do
+    before do
+      path = Temppath.mkdir
+      Util::Zip.uncompress(TestUtil::TEST_PACKAGE_DIR + "HelloWorld-gitrepos.zip", Location[path])
+      @location = Location[git: path]
+    end
+
+    behaves_like "package"
+  end
+end
+

data/test/component/spec_package.rb

@@ -2,23 +2,25 @@ require_relative '../test-util'
 
 describe "Pione::Component::Package" do
   before do
-    path = Location[File.expand_path("../spec_package", __FILE__)] + "TestPackage"
-    @package = Component::PackageReader.new(path).read
-    @scenario1 = @package.scenarios[0]
+    @location = TestUtil::Package.get("TestPackage1")
+    @package = Component::PackageReader.new(@location).read
+    @case1 = @package.scenarios[0]
+    @case2 = @package.scenarios[1]
+    @case3 = @package.scenarios[2]
   end
 
   it "should equal" do
-    Component::Package.new({"PackageName" => "Test"}, [], [], []).should ==
-      Component::Package.new({"PackageName" => "Test"}, [], [], [])
+    Component::Package.new(location: @location, info: {"PackageName1" => "Test"}).should ==
+      Component::Package.new(location: @location, info: {"PackageName1" => "Test"})
   end
 
   it "should not equal" do
-    Component::Package.new({"PackageName" => "Test1"}, [], [], []).should !=
-      Component::Package.new({"PackageName" => "Test2"}, [], [], [])
+    Component::Package.new(location: @location, info: {"PackageName1" => "Test1"}).should !=
+      Component::Package.new(location: @location, info: {"PackageName1" => "Test2"})
   end
 
   it "should get the package name" do
-    @package.name.should == "TestPackage"
+    @package.name.should == "TestPackage1"
   end
 
   it "should get bin" do
@@ -26,7 +28,10 @@ describe "Pione::Component::Package" do
   end
 
   it "should get scenarios" do
-    @package.scenarios.map{|scenario| scenario.name}.should.include "TestCase1"
+    names = @package.scenarios.map{|scenario| scenario.name}
+    names.should.include "Case1"
+    names.should.include "Case2"
+    names.should.include "Case3"
   end
 
   it "should get docuemtns" do
@@ -35,13 +40,13 @@ describe "Pione::Component::Package" do
 
   it "should get rules" do
     @package.rules.map{|rule| rule.path}.tap do |x|
-      x.should.include("&TestPackage:Main")
-      x.should.include("&TestPackage:Count")
+      x.should.include("&TestPackage1:Main")
+      x.should.include("&TestPackage1:Count")
     end
   end
 
   it "should get a main rule" do
-    @package.find_rule("Main").path.should == "&TestPackage:Main"
+    @package.find_rule("Main").path.should == "&TestPackage1:Main"
   end
 
   it "should upload package files" do
@@ -52,41 +57,53 @@ describe "Pione::Component::Package" do
   end
 
   it "should find sceinarios" do
-    @package.find_scenario(:anything).name.should == "TestCase1"
-    @package.find_scenario("TestCase1").name.should == "TestCase1"
+    @package.find_scenario(:anything).name.should == "Case1"
+    @package.find_scenario("Case1").name.should == "Case1"
+    @package.find_scenario("Case2").name.should == "Case2"
+    @package.find_scenario("Case3").name.should == "Case3"
   end
 end
 
 describe "Pione::Component::PackageScenario" do
   before do
-    @path = Location[File.expand_path("../spec_package", __FILE__)] + "TestPackage" + "scenario" + "case1"
-    @scenario = Component::PackageScenarioReader.new(@path).read
+    @location = TestUtil::Package.get("TestPackage1")
+    @case1 = Component::PackageScenarioReader.new(@location, "scenario/case1").read
+    @case2 = Component::PackageScenarioReader.new(@location, "scenario/case2").read
+    @case3 = Component::PackageScenarioReader.new(@location, "scenario/case3").read
   end
 
   it 'should equal' do
-    Component::PackageScenario.new(@path, {"ScenarioName" => "Test"}).should ==
-      Component::PackageScenario.new(@path, {"ScenarioName" => "Test"})
+    Component::PackageScenario.new(@location, "scenario/case1", {"ScenarioName" => "Test"}).should ==
+      Component::PackageScenario.new(@location, "scenario/case1", {"ScenarioName" => "Test"})
   end
 
   it 'should not equal' do
-    Component::PackageScenario.new(@path, {"ScenarioName" => "Test1"}).should !=
-      Component::PackageScenario.new(@path, {"ScenarioName" => "Test2"})
+    Component::PackageScenario.new(@location, "scenario/case1", {"ScenarioName" => "Test1"}).should !=
+      Component::PackageScenario.new(@path, "scenario/case1", {"ScenarioName" => "Test2"})
   end
 
   it "should get the scenario name" do
-    @scenario.name.should == "TestCase1"
-  end
-end
-
-describe "Pione::Component::PackageReader" do
-  it "should read package directory" do
-    path = Location[File.expand_path("../spec_package", __FILE__)] + "TestPackage"
-    Component::PackageReader.new(path).type.should == :directory
-  end
-
-  it "should read the package and return it" do
-    path = Location[File.expand_path("../spec_package", __FILE__)] + "TestPackage"
-    Component::PackageReader.read(path).should.kind_of Component::Package
+    @case1.name.should == "Case1"
+    @case2.name.should == "Case2"
+    @case3.name.should == "Case3"
+  end
+
+  it "should get input files" do
+    @case1.inputs[0].basename.should == "1.txt"
+    @case2.inputs[0].basename.should == "1.txt"
+    @case2.inputs[1].basename.should == "2.txt"
+    @case2.inputs[2].basename.should == "3.txt"
+    @case3.inputs[0].basename.should == "a.txt"
+    @case3.inputs[1].basename.should == "b.txt"
+  end
+
+  it "should get output files" do
+    @case1.outputs[0].basename.should == "1.count"
+    @case2.outputs[0].basename.should == "1.count"
+    @case2.outputs[1].basename.should == "2.count"
+    @case2.outputs[2].basename.should == "3.count"
+    @case3.outputs[0].basename.should == "a.count"
+    @case3.outputs[1].basename.should == "b.count"
   end
 end