dtk-dsl 1.0.2 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d8d7e9667524d13ccf2f97b3b150a435aa81f7ad
-  data.tar.gz: 2c1f1b937882c34122b10f160387da0a158be9b6
+  metadata.gz: 729eb1869724a0dfe75290f15ed8eb7735412797
+  data.tar.gz: 180a6b31a242f1b54f2c4b12e3f8464f87cca30d
 SHA512:
-  metadata.gz: 0bf7ed5a03b682eb4b908f4a979cec62b21fb213669afc03b28be701c82a258c91c62e49a20f4e93a732e74d9324e5444c773756b869751720e8853cd74a4596
-  data.tar.gz: e9eccc9fc8a61162631461dbdfdcd6b8d63279c0a9dfe8238e6f8b25e3fe08a5263ae7e6b47bfce77af5d8c6923d6bb7d0b7ee5582cd56f933f6610a85ac0416
+  metadata.gz: 04143ec146daa4c2157263e074f13ed75bf108e358c99c84b3ee13fd4d497ae5d962612d8d8d98e54aa9d28696283aa659e67db3b7e9b45afe603047ae230ea4
+  data.tar.gz: a3619c787e623b66ae81e3ad87b8e2874df66f1bb35d15713b1dbfd44eff0fca735085dba5df929a355c3da2046253b5f198002d449c2685d63f90183ed9ca0f

data/lib/dsl/input_output_common/canonical/hash_key.rb

@@ -21,6 +21,7 @@ module DTK::DSL
       # Top level for common module
       DSLVersion       = :dsl_version
       Assemblies       = :assemblies
+      ComponentDefs    = :component_defs
       DependentModules = :dependent_modules
       ModuleRef        = :module
 

data/lib/dsl/input_output_common/semantic_parse/hash.rb

@@ -26,6 +26,11 @@ module DTK::DSL
           super(parent_class)
           initialize_semantic_parse(opts)      
         end
+
+        # The method skip_for_generation? can be overwritten
+        def skip_for_generation?
+          false
+        end
       end
     end
   end

data/lib/dsl/input_output_common/semantic_parse/mixin.rb

@@ -27,7 +27,7 @@ module DTK::DSL
         private :initialize_semantic_parse
 
         def qualified_key
-          @qualified_key || fail(Error, "Unexepcetd that @qualified_key is nil")
+          @qualified_key || fail(Error, "Unexepected that @qualified_key is nil")
         end
 
         def name 

data/lib/dsl/service_and_component_info/info/input_files.rb

@@ -0,0 +1,58 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK::DSL
+  class ServiceAndComponentInfo::Info
+    class InputFiles
+      def initialize(regexps)
+        @regexps = regexps
+
+        # dyanmically set
+        # For TransformFrom hash_content is type ::Hash where for TransformTo it is type InputOutputCommon::Canonical::Hash
+        @ndx_file_hash_content = {} #indexed by file hash name
+      end
+
+      def match?(path)
+        @regexps.find { |regexp| path =~ regexp }
+      end
+
+      def content_hash_array
+        @ndx_file_hash_content.values
+      end
+
+      def input_paths
+        @ndx_file_hash_content.keys
+      end
+
+      def empty?
+        @ndx_file_hash_content.empty?
+      end
+
+      def content_hash?
+        fail Error, "Unexpected that @ndx_file_hash_content.size > 1" if @ndx_file_hash_content.size > 1
+        @ndx_file_hash_content.values.first
+      end
+
+      private
+
+      def add_hash_content!(path, hash_content)
+        @ndx_file_hash_content.merge!(path => hash_content)
+      end
+
+    end
+  end
+end

data/lib/dsl/service_and_component_info/info.rb

@@ -0,0 +1,64 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK::DSL
+ class ServiceAndComponentInfo
+    class Info
+      require_relative('info/input_files')
+
+      attr_reader :indexed_input_files, :module_ref
+      def initialize(parent)
+        @parent      = parent
+        @module_ref  = parent.module_ref
+        # indexed by input file type
+        @indexed_input_files = ret_indexed_input_files(info_type)
+      end
+
+      def compute_outputs!        
+        raise Error::NoMethodForConcreteClass.new(self.class)
+      end
+
+      def output_path_text_pairs
+        @parent.output_path_text_pairs
+      end
+
+      private
+
+      # indexed by input file type
+      def ret_indexed_input_files(input_type)
+        input_type_hash(input_type)[:input_files].inject({}) { |h, (k, v) | h.merge(k => input_files_class.new(v[:regexps])) }
+      end
+
+      def input_type_hash(input_type)
+        self.class::INFO_HASH[input_type] || fail(Error, "Illegal input_type '#{input_type}'")
+      end
+
+      def output_file_hash(path)
+        @parent.output_path_hash_pairs[path] || {}
+      end
+
+      def update_or_add_output_hash!(path, hash_content)
+        @parent.update_or_add_output_hash!(path, hash_content)
+      end
+
+      def input_files_class
+        @input_files_class ||= @parent.class::Info::InputFiles
+      end
+
+    end
+  end
+end

data/lib/dsl/service_and_component_info/module_ref.rb

@@ -0,0 +1,56 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK::DSL
+  class ServiceAndComponentInfo
+    class ModuleRef
+      attr_reader :namespace, :module_name, :version
+      def initialize(namespace, module_name, version)
+        @namespace = namespace
+        @module_name = module_name
+        @version     = version || 'master'
+      end
+      
+      NAMESPACE_NAME_DELIM = '/'
+      
+      def combined_module_form
+        { "#{@namespace}#{NAMESPACE_NAME_DELIM}#{@module_name}" => @version }
+      end
+      
+      def raise_error_if_conflict(ndx_existing_modules)
+        if matching_module_info = ndx_existing_modules[@module_name]
+          fail Error::Usage, conflict_error_msg(matching_module_info) unless match?(matching_module_info)
+        end
+      end
+      
+      def print_form
+        "#{@namespace}#{NAMESPACE_NAME_DELIM}#{@module_name}(#{@version})"
+      end
+      
+      private
+      
+      def match?(module_info2)
+        @version == module_info2.version and @module_name == module_info2.module_name and @namespace == module_info2.namespace
+      end
+      
+      def conflict_error_msg(matching_module_info)
+        "Conflicting versions of module '#{@module_name}': '#{print_form}' vs '#{matching_module_info.print_form}'"
+      end
+        
+    end
+  end
+end

data/lib/dsl/service_and_component_info/parser.rb

@@ -0,0 +1,58 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK::DSL
+  class ServiceAndComponentInfo
+    class Parser
+      
+      attr_reader :output_hash, :info_object
+      def initialize(output_hash, info_object)
+        @output_hash         = output_hash
+        @info_object         = info_object
+        @indexed_input_files = info_object.indexed_input_files
+      end
+      private :initialize
+      
+      def self.update_output_hash?(output_hash, info_object)
+        new(output_hash, info_object).update_output_hash?
+        output_hash
+      end
+      
+      private
+      
+      def input_files(type)
+        input_files?(type) || raise(Error, "Unexpected that no indexed_input_files of type '#{type}'")
+      end
+      
+      def input_files?(type)
+        @indexed_input_files[type]
+      end
+
+      def input_file_hash?(type)
+        if input_files = input_files?(type)
+          input_files.content_hash?
+        end
+      end
+      
+      def raise_error_missing_field(key)
+        raise Error, "Unexpected that field '#{key}' is missing"
+      end
+      
+    end
+  end
+end
+

data/lib/dsl/service_and_component_info/transform_from/info/component.rb

@@ -0,0 +1,42 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK::DSL
+  class ServiceAndComponentInfo::TransformFrom
+    class Info
+      class Component < self
+        def compute_outputs!
+          path = top_level_dsl_path
+          update_or_add_output_hash!(path, top_dsl_file_hash_content!(output_file_hash(path)))
+        end
+        
+        private
+        
+        def info_type
+          :component_info
+        end
+
+        def top_dsl_file_hash_content!(output_hash)
+          top_dsl_parser::ModuleInfo.update_output_hash?(output_hash, self) 
+          top_dsl_parser::Dependencies.update_output_hash?(output_hash, self)
+          top_dsl_parser::Components.update_output_hash?(output_hash, self)
+          output_hash
+        end
+      end
+    end
+  end
+end

data/lib/dsl/service_and_component_info/transform_from/info/input_files.rb

@@ -0,0 +1,27 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK::DSL
+  class ServiceAndComponentInfo::TransformFrom::Info
+    class InputFiles < ServiceAndComponentInfo::Info::InputFiles
+      def add_content!(path, text_content)
+        file_obj = FileObj.new(nil, path, :content => text_content)
+        add_hash_content!(path, YamlHelper.parse(file_obj))
+      end
+    end
+  end
+end

data/lib/dsl/service_and_component_info/transform_from/info/service.rb

@@ -0,0 +1,43 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK::DSL
+  class ServiceAndComponentInfo::TransformFrom
+    class Info
+      class Service < self
+        def compute_outputs!
+          path = top_level_dsl_path
+          update_or_add_output_hash!(path, top_dsl_file_hash_content!(output_file_hash(path)))
+        end
+      
+        private
+      
+        def info_type
+          :service_info
+        end
+
+        def top_dsl_file_hash_content!(output_hash)
+          top_dsl_parser::ModuleInfo.update_output_hash?(output_hash, self) 
+          top_dsl_parser::Dependencies.update_output_hash?(output_hash, self)
+          top_dsl_parser::Assemblies.update_output_hash?(output_hash, self)
+          output_hash
+        end
+
+      end
+    end
+  end
+end

data/lib/dsl/service_and_component_info/transform_from/info.rb

@@ -0,0 +1,64 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK::DSL
+ class ServiceAndComponentInfo::TransformFrom
+    class Info < ServiceAndComponentInfo::Info
+      require_relative('info/input_files')
+      require_relative('info/service')
+      require_relative('info/component')
+      # This is for mapping to module directories (not service instance directories)
+      INFO_HASH = {
+        :service_info => {
+          :input_files => {
+            :assemblies => {
+              :regexps => 
+              [
+               Regexp.new("assemblies/(.*)\.dtk\.assembly\.(yml|yaml)$"), 
+               Regexp.new("assemblies/([^/]+)/assembly\.(yml|yaml)$") #Legacy form
+              ]
+            },
+            :module_refs =>  {
+              :regexps => [Regexp.new("module_refs\.yaml$")]
+            }
+          }
+        },
+        :component_info => {
+          :input_files => {
+            :component_dsl_file => {
+              :regexps => [Regexp.new("dtk\.model\.yaml$")], 
+            },
+            :module_refs =>  {
+              :regexps => [Regexp.new("module_refs\.yaml$")]
+            }
+          }
+        }
+      }
+
+      private
+
+      def top_dsl_parser
+        @top_dsl_parser ||= Parser::TopDSL
+      end
+
+      def top_level_dsl_path  
+        @top_level_dsl_path ||= FileType::CommonModule::DSLFile::Top.canonical_path
+      end
+
+    end
+  end
+end

data/lib/dsl/service_and_component_info/transform_from/parser/top_dsl/assemblies/node_bindings.rb

@@ -0,0 +1,81 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK::DSL
+  class ServiceAndComponentInfo::TransformFrom::Parser
+    class TopDSL::Assemblies 
+      module NodeBindings
+        def self.add_node_properties!(assembly_content, node_bindings) 
+          nodes = assembly_content['nodes'] || []
+          return if nodes.empty?
+        
+          node_bindings.each do |node, node_binding|
+            if node_content = nodes[node]
+              process_node_content!(node_content, node_binding) 
+            end
+          end
+        end
+
+        private
+
+        def self.process_node_content!(node_content, node_binding)
+          components  = node_content['components']
+          components  = components.is_a?(Array) ? components : [components]
+          image, size = ec2_properties_from_node_binding(node_binding)
+          new_attrs   = { 'image' => image, 'size' => size }
+
+          if index = include_node_property_component?(components)
+            ec2_properties = components[index]
+            if ec2_properties.is_a?(::Hash)
+              if attributes = ec2_properties.values.first['attributes']
+                attributes['image'] ||= image
+                attributes['size'] ||= size
+              else
+                ec2_properties.merge!('attributes' => new_attrs)
+              end
+            else
+              components[index] = { ec2_properties => { 'attributes' => new_attrs } }
+            end
+          elsif node_attributes = node_content['attributes']
+            node_attributes['image'] ||= image 
+            node_attributes['size'] ||= size 
+          else
+            node_content['attributes'] = new_attrs
+          end
+        end
+
+        def self.ec2_properties_from_node_binding(node_binding)
+          image, size = node_binding.split('-')
+          [image, size]
+        end
+
+        def self.include_node_property_component?(components)
+          property_component = 'ec2::properties'
+          components.each do |component|
+            if component.is_a?(Hash)
+              return components.index(component) if component.keys.first.eql?(property_component)
+            else
+              return components.index(component) if component.eql?(property_component)
+            end
+          end
+          false
+        end
+
+      end
+    end
+  end
+end

data/lib/dsl/service_and_component_info/transform_from/parser/top_dsl/assemblies/workflows.rb

@@ -0,0 +1,37 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK::DSL
+  class ServiceAndComponentInfo::TransformFrom::Parser
+    class TopDSL::Assemblies 
+      module Workflows
+        def self.hash_content?(assembly_hash)
+          if workflows = assembly_hash['workflow'] || assembly_hash['workflows']
+            if workflow_name = workflows['assembly_action']
+              # this is legacy workflow
+              workflows_without_name = workflows.inject({}) { |h, (k, v)| k == 'assembly_action' ? h : h.merge(k => v) }
+              { workflow_name =>  workflows_without_name }
+            else
+              workflows
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/dsl/service_and_component_info/transform_from/parser/top_dsl/assemblies.rb

@@ -0,0 +1,52 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK::DSL
+  class ServiceAndComponentInfo::TransformFrom::Parser
+    class TopDSL
+      class Assemblies < self
+        require_relative('assemblies/workflows')
+        require_relative('assemblies/node_bindings')
+        def update_output_hash?
+          input_files(:assemblies).content_hash_array.each do |input_hash|
+            name          = input_hash['name'] || raise_error_missing_field('name')
+            assembly_hash = input_hash['assembly'] || raise_error_missing_field('assembly')
+
+            assembly_content = {}
+            if description  = input_hash['description']
+              assembly_content.merge!('description' => description)
+            end
+            
+            assembly_content.merge!(assembly_hash)
+            
+            if workflows = Workflows.hash_content?(input_hash)
+              assembly_content.merge!('workflows' => workflows)
+            end
+
+            if node_bindings = input_hash['node_bindings']
+              # convert node_bindings to node attributes
+              NodeBindings.add_node_properties!(assembly_content, node_bindings) 
+            end
+            (output_hash['assemblies'] ||= {}).merge!(name => assembly_content)
+          end
+
+        end
+
+      end
+    end
+  end
+end

data/lib/dsl/service_and_component_info/transform_from/parser/top_dsl/components.rb

@@ -0,0 +1,33 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK::DSL
+  class ServiceAndComponentInfo::TransformFrom::Parser
+    class TopDSL
+      class Components < self
+        def update_output_hash?
+          if component_dsl_input_hash = input_file_hash?(:component_dsl_file)
+            if component_defs = component_dsl_input_hash['components']
+              output_hash['component_defs'] = component_defs
+            end
+          end
+        end
+
+      end
+    end
+  end
+end