pione 0.2.2 → 0.3.0

data/lib/pione/package/package-filename.rb

@@ -0,0 +1,46 @@
+module Pione
+  module Package
+    class PackageFilenameParser < Parslet::Parser
+      root(:filename)
+
+      rule(:filename) { package_name.maybe >> editor.maybe >> tag.maybe >> digest.maybe >> ext.maybe }
+
+      rule(:package_name) { ((str("(") | str(".") | str("+") | str("@")).absent? >> any).repeat(1).as(:package_name) }
+      rule(:editor) { str("(") >> (str(")").absent? >> any).repeat(1).as(:editor) >> str(")") }
+      rule(:tag) { str("+") >> ((ext | str("@")).absent? >> any).repeat(1).as(:tag) }
+      rule(:digest) { str("@") >>  match('[0-9a-fA-F]').repeat(1).as(:digest) }
+      rule(:ext) { str(".ppg") }
+    end
+
+    class PackageFilename < StructX
+      member :package_name
+      member :editor, default: "origin"
+      member :tag
+      member :digest
+
+      class << self
+        # Parse the filename.
+        def parse(str)
+          begin
+            new(PackageFilenameParser.new.parse(str))
+          rescue => e
+            raise InvalidPackageFilename.new(str, e)
+          end
+        end
+      end
+
+      def string(ext=true)
+        name = ""
+        name << package_name
+        name << "(%s)" % editor if editor and editor != "origin"
+        name << "+%s" % tag if tag
+        name << "@%s" % digest if digest
+        name << ".ppg" if ext
+        return name
+      end
+
+      alias :to_s :string
+    end
+  end
+end
+

data/lib/pione/package/package-handler.rb

@@ -0,0 +1,106 @@
+module Pione
+  module Package
+    # PackageHandler is a set of operations for package.
+    class PackageHandler
+      class << self
+        # Update package info file and scenario info files.
+        def write_info_files(location)
+          package_info = PackageScanner.new(location).scan
+          (location + "pione-package.json").write(JSON.pretty_generate(package_info))
+          Log::SystemLog.info("write %s" % (location + "pione-package.json").address)
+          package_info.scenarios.each do |scenario|
+            scenario_info = ScenarioScanner.new(location + scenario).scan
+            ScenarioHandler.new(location + scenario, scenario_info).write_info_file
+          end
+        end
+      end
+
+      attr_reader :location # local package location
+      attr_reader :info
+      attr_reader :digest
+      attr_reader :package_type
+
+      def initialize(location, key={})
+        @location = location
+
+        unless key.has_key?(:digest) or key.has_key?(:info)
+          raise ArgumentError
+        end
+
+        # init with digest
+        if digest = key[:digest]
+          @digest = digest
+          json = (PackageCache.directory_cache(digest) + "pione-package.json").read
+          @info = PackageInfo.read(json)
+        end
+
+        # init with info for single document location
+        if info = key[:info]
+          @digest = nil
+          @info = info
+        end
+      end
+
+      # Evaluate the package context in the environment. This method will
+      # introduce a new package id, and the context is evaluated under it.
+      def eval(env)
+        # load parent packages
+        parent_package_ids = @info.parents.map do |parent_info|
+          load_parent_package(env, parent_info.name, parent_info.editor, parent_info.tag)
+        end
+
+        # load this package
+        _env = env.setup_new_package(info.name || "Anonymous", parent_package_ids)
+        @info.documents.inject(Lang::PackageContext.new) do |_context, d|
+          Document.load(_env, @location + d, @info.name, @info.editor, @info.tag, d)
+        end
+
+        return _env
+      end
+
+      # Upload the package files to the location.
+      def upload(dest)
+        @info.bins.each do |entry|
+          (@location + entry).copy(dest + entry)
+        end
+      end
+
+      # Find scenario that have the name.
+      #
+      # @param name [String]
+      #   scenario name
+      # @return [ScenarioHandler]
+      #   the scenario handler
+      def find_scenario(name)
+        @info.scenarios.each do |path|
+          if name == :anything
+            # read first hit scenario
+            return ScenarioReader.read(@location + path)
+          else
+            # read matched scenario
+            scenario = ScenarioReader.read(@location + path)
+            if scenario.info.name == name
+              return scenario
+            end
+          end
+        end
+      end
+
+      private
+
+      # Load parent packages from package database. Parent packages should be
+      # recorded in the database, or Package::NotFound error is raised.
+      def load_parent_package(env, name, editor, tag)
+        if digest = Global.package_database.find(name, editor, tag)
+          parent_location = PackageCache.directory_cache(digest)
+          handler = PackageHandler.new(parent_location, digest: digest)
+          handler.eval(env)
+          return env.current_package_id
+        else
+          raise NotFound.new(name, editor, tag)
+        end
+      end
+    end
+  end
+end
+

data/lib/pione/package/package-info.rb

@@ -0,0 +1,72 @@
+module Pione
+  module Package
+    # PackageInfo is a information storage about package.
+    class PackageInfo < StructX
+      member :name
+      member :editor
+      member :tag
+      member :parents, default: lambda { Array.new }
+      member :documents, default: lambda { Array.new }
+      member :scenarios, default: lambda { Array.new }
+      member :bins, default: lambda { Array.new }
+
+      # Read package information from the string.
+      def self.read(str)
+        data = JSON.load(str)
+        args = Hash.new
+        args[:name] = data["PackageName"] if data.has_key?("PackageName")
+        args[:editor] = data["Editor"] if data.has_key?("Editor")
+        args[:tag] = data["Tag"] if data.has_key?("Tag")
+        args[:parents] = data["Parents"].map {|_data| ParentPackageInfo.json_create(_data)}
+        args[:documents] = data["Documents"]
+        args[:scenarios] = data["Scenarios"]
+        args[:bins] = data["Bins"]
+        new(args)
+      end
+
+      # Return package file paths.
+      def filepaths
+        list = []
+        list += documents
+        list += bins
+        return list
+      end
+
+      # Convert to JSON object.
+      def to_json(*args)
+        data = {}
+        data["PackageName"] = name
+        data["Editor"] = editor if editor
+        data["Tag"] = tag if tag
+        data["Parents"] = parents
+        data["Documents"] = documents
+        data["Scenarios"] = scenarios
+        data["Bins"] = bins
+        data.to_json(*args)
+      end
+    end
+
+    # ParentPackageInfo is a information set about parent package.
+    class ParentPackageInfo < StructX
+      member :name
+      member :editor
+      member :tag
+
+      def self.json_create(data)
+        args = Hash.new
+        args[:name] = data["PackageName"]
+        args[:editor] = data["Editor"] if data.has_key?("Editor")
+        args[:tag] = data["Tag"] if data.has_key?("Tag")
+        new(args)
+      end
+
+      def to_json
+        data = {}
+        data["PackageName"] = name
+        data["Editor"] = editor if editor
+        data["Tag"] = tag if tag
+        data.to_json
+      end
+    end
+  end
+end

data/lib/pione/package/package-reader.rb

@@ -0,0 +1,202 @@
+module Pione
+  module Package
+    # PackageReader is a reader for packages.
+    class PackageReader
+      class << self
+        # Read a pacakge from the location.
+        def read(location)
+          case PackageTypeClassifier.classify(location)
+          when :directory
+            DirectoryPackageReader.new(location).read
+          when :git
+            GitPackageReader.new(location).read
+          when :archive
+            ArchivePackageReader.new(location).read
+          when :document
+            SingleDocumentPackageReader.new(location).read
+          end
+        end
+      end
+
+      def initialize(location)
+        @location = location
+        @local = nil
+        @package_info = read_package_info
+      end
+
+      def read
+        raise NotImplementedError
+      end
+
+      private
+
+      # Read informations from the package location.
+      def read_package_info
+        PackageInfo.load((@location + "pione-package.json").read)
+      rescue Location::NotFound
+        raise InvalidPackage.new(self, "pione-package.json not found in %s" % @location.uri)
+      end
+
+      #
+      # @return [Package]
+      #   the package
+      def read_document
+        opt = {package_name: "Anonymous", filename: @location.basename}
+        context = Document.load(@location, opt)
+        Package.new(info: {"PackageName" => "Anonymous"}, context: context)
+      end
+
+      # Find scenarios from the package location.
+      #
+      # @return [Array<PackageScenario>]
+      #   scenarios
+      def find_scenario_paths(scenarios)
+        return [] if scenarios.nil?
+        scenarios.select do |path|
+          (@location + path + "scenario.yml").exist?
+        end.uniq.compact
+      end
+    end
+
+    # PackageTypeClassifier provides the function to distinguish package types.
+    module PackageTypeClassifier
+      class << self
+        # Distinguish the type of package based on the location.
+        #
+        # @param [BasicLocation]
+        #   location of the package
+        # @return [Symbol]
+        #   package type
+        def classify(location)
+          return :git if git?(location)
+          return :archive if archive?(location)
+          return :document if document?(location)
+          return :directory
+        end
+
+        private
+
+        # Return true if the location represents git package.
+        def git?(location)
+          location.location_type == :git_repository
+        end
+
+        # Return true if the location represents archive package.
+        def archive?(location)
+          location.file? and location.extname == ".ppg"
+        end
+
+        # Return true if the location represents document package.
+        def document?(location)
+          location.file? and location.extname == ".pione"
+        end
+      end
+    end
+
+    # DirectoryPackageReader reads package structure from directory based
+    # package.
+    class DirectoryPackageReader < PackageReader
+      def initialize(location)
+        @location = location
+      end
+
+      def read
+        # copy to local
+        local_location = make_local_location
+
+        # cache
+        digest = PackageCache.cache(local_location)
+
+        PackageHandler.new(PackageCache.directory_cache(digest), digest: digest)
+      end
+
+      def make_local_location
+        # make temporary directory
+        local_location = Location[Temppath.create]
+
+        # copy files to local
+
+        # pione-package.json
+        info = PackageInfo.read((@location + "pione-package.json").read)
+        (@location + "pione-package.json").copy(local_location + "pione-package.json")
+
+        # documents
+        (info.documents + info.bins).each do |path|
+          (@location + path).copy(local_location + path)
+        end
+
+        # scenarios
+        info.scenarios.each do |path|
+          scenario_info = ScenarioInfo.read((@location + path + "pione-scenario.json").read)
+          (@location + path + "pione-scenario.json").copy(local_location + path + "pione-scenario.json")
+          (@location + path + "Scenario.pione").copy(local_location + path + "Scenario.pione")
+          (scenario_info.inputs + scenario_info.outputs).each do |_path|
+            (@location + path + _path).copy(local_location + path + _path)
+          end
+        end
+
+        return local_location
+      rescue Location::LocationError => e
+        raise InvalidPackage.new("package \"%s\" is invalid: %s" % [@location.address, e.message])
+      end
+    end
+
+    # GitPackageReader is a reader for git based package.
+    class GitPackageReader < PackageReader
+      def initialize(location)
+        @location = location
+      end
+
+      def read
+        local_location = make_local_location
+        digest = PackageCache.cache(local_location)
+        return PackageHandler.new(PackageCache.directory_cache(digest), digest: digest)
+      end
+
+      private
+
+      def make_local_location
+        # we should use local cloned repository, but currently export
+        local_location = Location[Temppath.mkdir]
+        @location.export(local_location)
+        return local_location
+      end
+    end
+
+    # ArchivePackageReader is a reader for PPG pakage.
+    class ArchivePackageReader < PackageReader
+      def initialize(location)
+        @location = location # arbitrary data location
+      end
+
+      def read
+        # copy to local, we cannot use Location::DataLocation#local in here
+        # because the method don't keep filename
+        local_location = Location[Temppath.mkdir] + @location.basename
+        @location.copy(local_location)
+
+        # cache the package
+        digest = PackageCache.cache(local_location)
+
+        return PackageHandler.new(PackageCache.directory_cache(digest), digest: digest)
+      end
+    end
+
+    # SingleDocumentPackageReader is a reader for single document package.
+    class SingleDocumentPackageReader < PackageReader
+      def initialize(location)
+        @location = location
+      end
+
+      def read
+        # copy to local
+        local_location = @location.local
+
+        # make package info by scanning annotations
+        info = PackageScanner.new(local_location).scan
+
+        return PackageHandler.new(local_location.dirname, info: info)
+      end
+    end
+  end
+end

data/lib/pione/package/package-scanner.rb

@@ -0,0 +1,133 @@
+module Pione
+  module Package
+    class PackageScanner
+      class << self
+        def scan(location)
+          new(location).scan
+        end
+      end
+
+      def initialize(location)
+        @package_location = location
+      end
+
+      # Scan the package location and return package informations.
+      def scan
+        if @package_location.directory?
+          documents = scan_documents(@package_location)
+          name, editor, tag, parents = scan_annotations(documents)
+          scenarios = scan_scenarios(@package_location)
+          bins = scan_bins(@package_location)
+
+          return PackageInfo.new(
+            name: name, editor: editor, tag: tag, parents: parents,
+            documents: documents, scenarios: scenarios, bins: bins
+          )
+        else
+          # the case for single document package
+          name, editor, tag, parents = scan_annotations([@package_location])
+          documents = [@package_location.basename]
+
+          return PackageInfo.new(
+            name: name, editor: editor, tag: tag, parents: parents,
+            documents: documents
+          )
+        end
+      end
+
+      private
+
+      # Scan PIONE documents in this package. This scans each directory
+      # excluding scenarios in the location recursively.
+      def scan_documents(location)
+        location.entries.each_with_object([]) do |entry, paths|
+          if entry.file?
+            # PIONE document has extension ".pione"
+            if /^[^.].+.pione$/.match(entry.basename)
+              # document path should be relative
+              paths << entry.path.relative_path_from(@package_location.path).to_s
+            end
+          else
+            # excepting dot-headed or scenario, scan the directory recursively
+            unless /^\./.match(entry.basename) or ScenarioScanner.scenario?(entry)
+              paths.concat(scan_documents(entry))
+            end
+          end
+        end
+      end
+
+      # Scan annotations from documents.
+      def scan_annotations(documents)
+        # setup temporary language environment for reading annotations
+        env = Lang::Environment.new.setup_new_package("PackageScanner")
+
+        # read documents
+        documents.each do |doc|
+          location = doc.is_a?(Location::DataLocation) ? doc : @package_location + doc
+          Document.load(env, location, nil, nil, nil, doc)
+        end
+
+        # get annotations
+        annotations = env.package_get(Lang::PackageExpr.new(package_id: env.current_package_id)).annotations
+        name = find_package_info(annotations, "PackageName")
+        editor = find_package_info(annotations, "Editor")
+        tag = find_package_info(annotations, "Tag")
+        parents = find_parents(annotations)
+
+        return name, editor, tag, parents
+      end
+
+      # Find package information of the annotation type from annotations.
+      def find_package_info(annotations, annotation_type)
+        annotations.each do |annotation|
+          if annotation.annotation_type == annotation_type
+            return annotation.pieces.first.value
+          end
+        end
+
+        return nil
+      end
+
+      # Find parents from annotaions.
+      def find_parents(annotations)
+        annotations.each_with_object([]) do |annotation, parents|
+          if annotation.annotation_type == "Parent"
+            annotation.pieces.each do |parent|
+              name = parent.name
+              editor = parent.editor ? parent.editor.value : nil
+              tag = parent.tag ? parent.tag.value : nil
+              parents << ParentPackageInfo.new(name: name, editor: editor, tag: tag)
+            end
+          end
+        end
+      end
+
+      # Scan scenarios recursively. See ScnarioScanner for details of scenario
+      # directory.
+      def scan_scenarios(location)
+        location.entries.each_with_object([]) do |entry, list|
+          if entry.directory?
+            if ScenarioScanner.scenario?(entry)
+              list << entry.path.relative_path_from(@package_location.path).to_s
+            else
+              list.concat(scan_scenarios(entry))
+            end
+          end
+        end
+      end
+
+      # Scan script files.
+      def scan_bins(location)
+        if (location + "bin").exist?
+          (location + "bin").entries.each_with_object([]) do |entry, list|
+            if entry.file?
+              list << entry.path.relative_path_from(@package_location.path).to_s
+            end
+          end
+        else
+          return []
+        end
+      end
+    end
+  end
+end