hq-engine 0.0.13 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dcb4e7698ec7f5dea42319d06693715c8298e09b
-  data.tar.gz: 19db413442a4ab4a3474e9d67ec2c427beb658fd
+  metadata.gz: 3dfe1ba890c382485213a074683030ee06736e54
+  data.tar.gz: c75fa82d77e182ab2b243efcf9aebaa3799e4adb
 SHA512:
-  metadata.gz: 4ecea554285470f2bc6e86640bbe889a1e6f2f97bb9370c3664e412b2cce816a115789df44f625b966977e938ffb61c925b0304eddd955a159d3f83133567d9c
-  data.tar.gz: d6a3f189fb6e347ebcacf69cf7cf6fda4982a92d18d6f101ae4aec2169067532bff1203077d5bd4e28ddf228cbd546b3d54c40f693c8a02a641d728127409344
+  metadata.gz: 4d39b500ce568c06e775d08e0714f580a183febb3421916571f8042220e2ab8bbf2aa1c1187ecfd58f017b8b684ca17f79518ffcc1e764170c90513e4636bcb2
+  data.tar.gz: abff7975df830b6ecd77c33776d95a76e2fa6e3b1f12aa0f06708fcb0f1a9331f8bffaae4e1545e7e91aeb082b945a796a37c4c4a3a5f2a753b7883685c35daa

data/features/engine/engine.feature

@@ -0,0 +1,128 @@
+Feature: Engine
+
+  Background:
+
+    Given a file "config/rules/input-schema.rb":
+      """
+      # (: out schema :)
+      require "xml"
+      proc do |hq|
+        doc = XML::Document.string '
+          <schema name="input">
+            <id><text name="name"/></id>
+            <fields><text name="value"/></fields>
+          </schema>
+        '
+        hq.write doc.root
+      end
+      """
+
+    And a file "config/rules/schema-schema.rb":
+      """
+      # (: out schema :)
+      require "xml"
+      proc do |hq|
+        doc = XML::Document.string '
+          <schema name="schema">
+            <id><text name="name"/></id>
+            <fields/>
+          </schema>
+        '
+        hq.write doc.root
+      end
+      """
+
+    And a file "config/rules/output-schema.rb":
+      """
+      # (: out schema :)
+      require "xml"
+      proc do |hq|
+        doc = XML::Document.string '
+          <schema name="output">
+            <id><text name="name"/></id>
+            <fields><text name="value"/></fields>
+          </schema>
+        '
+        hq.write doc.root
+      end
+      """
+
+    And a file "config/rules/input-to-output.rb":
+      """
+      # (: in input :)
+      # (: out output :)
+      require "xml"
+      proc do |hq|
+        hq.find("input").each do
+          |input|
+          output = XML::Node.new "output"
+          output["name"] = input["name"]
+          output["value"] = input["value"].upcase
+          hq.write output
+        end
+      end
+      """
+
+    And a file "work/input.yaml":
+      """
+      - id: input/name
+        type: input
+        value:
+          name: name
+          value: value
+      """
+
+    And a file "default.args":
+      """
+      --config config
+      --work work
+      """
+
+  Scenario: Success
+
+    When I invoke engine with "default.args"
+
+    Then there should be a file "work/output/data/output/name.xml":
+      """
+      <output name="name" value="VALUE"/>
+      """
+
+    Then there should be a file "work/output/data/schema/schema.xml":
+      """
+      <schema name="schema">
+        <id>
+          <text name="name"/>
+        </id>
+        <fields/>
+      </schema>
+      """
+
+    Then there should be a file "work/output/data/schema/output.xml":
+      """
+      <schema name="output">
+        <id>
+          <text name="name"/>
+        </id>
+        <fields>
+          <text name="value"/>
+        </fields>
+      </schema>
+      """
+
+    Then there should be a file "work/output/data/schema/input.xml":
+      """
+      <schema name="input">
+        <id>
+          <text name="name"/>
+        </id>
+        <fields>
+          <text name="value"/>
+        </fields>
+      </schema>
+      """
+
+    Then there should be a file "work/output/data/input/name.xml":
+      """
+      <input name="name" value="value"/>
+      """
+

data/features/support/env.rb

@@ -0,0 +1,10 @@
+require "hq/cucumber/command"
+require "hq/cucumber/temp-dir"
+
+require "hq/engine/engine-script"
+require "hq/transform/transform-script"
+
+include HQ
+
+$commands["engine"] = Engine::EngineScript
+$commands["transform"] = Transform::TransformScript

data/features/transform/transform.feature

@@ -0,0 +1,111 @@
+Feature: Transformer
+
+  The transformer takes a schema, a set of data files, and a list of rules, and
+  it produces a structured data tree as output. It depends on a backend which
+  must be provided externally.
+
+  Background:
+
+    Given a file "schema.xml":
+      """
+      <data>
+        <schema name="input">
+          <id>
+            <text name="name"/>
+          </id>
+          <fields>
+            <text name="value"/>
+          </fields>
+        </schema>
+        <schema name="internal">
+          <id>
+            <text name="name"/>
+          </id>
+          <fields>
+            <text name="value"/>
+          </fields>
+        </schema>
+        <schema name="output">
+          <id>
+            <text name="name"/>
+          </id>
+          <fields>
+            <text name="value"/>
+          </fields>
+        </schema>
+      </data>
+      """
+
+    And a file "input/data.xml":
+      """
+      <data>
+        <input name="name" value="value"/>
+      </data>
+      """
+
+    And a file "rules/input-to-internal.rb":
+      """
+      # (: in input :)
+      # (: out internal :)
+      require "xml"
+      include XML
+      proc do
+        |hq|
+        hq.find("input").each do
+          |input|
+          internal = Node.new "internal"
+          internal["name"] = input["name"]
+          internal["value"] = input["value"].upcase
+          hq.write internal
+        end
+      end
+      """
+
+    And a file "rules/internal-to-output.rb":
+      """
+      # (: in internal :)
+      # (: out output :)
+      require "xml"
+      include XML
+      proc do
+        |hq|
+        hq.find("internal").each do
+          |internal|
+          2.times do
+            |time|
+            output = Node.new "output"
+            output["name"] = "#{internal["name"]}-#{time}"
+            output["value"] = internal["value"]
+            hq.write output
+          end
+        end
+      end
+      """
+
+    And a file "default.args":
+      """
+      --schema schema.xml
+      --input input
+      --rules rules
+      --include include
+      --output output
+      """
+
+  Scenario: Success
+
+    When I invoke transform with "default.args"
+
+    Then there should be a file "output/data/input/name.xml":
+      """
+      <input name="name" value="value"/>
+      """
+
+    And there should be a file "output/data/internal/name.xml":
+      """
+      <internal name="name" value="VALUE"/>
+      """
+
+    And there should be a file "output/data/output/name-1.xml":
+      """
+      <output name="name-1" value="VALUE"/>
+      """

data/lib/hq/engine/engine-script.rb

@@ -0,0 +1,66 @@
+require "hq/logger"
+require "hq/tools/base-script"
+require "hq/tools/getopt"
+require "hq/engine/engine"
+require "hq/transform/ruby-backend"
+
+module HQ
+module Engine
+
+class EngineScript < Tools::BaseScript
+
+  attr_accessor :logger
+  attr_accessor :transform_backend
+
+  def initialize
+
+    @logger = Logger.new
+    @logger.hostname = "local"
+    @logger.add_auto "debug"
+
+    @transform_backend = Transform::RubyBackend.new
+
+  end
+
+  def main
+    process_args
+    run_engine
+  end
+
+  def process_args
+
+    @opts, @args =
+      Tools::Getopt.process @args, [
+
+        { :name => :config,
+          :required => true },
+
+        { :name => :work,
+          :required => true },
+
+      ]
+
+    @args.empty? \
+      or raise "Extra args on command line"
+
+  end
+
+  def run_engine
+
+    engine =
+      Engine.new
+
+    engine.logger = logger
+    engine.transform_backend = transform_backend
+
+    engine.config_dir = @opts[:config]
+    engine.work_dir = @opts[:work]
+
+    engine.transform
+
+  end
+
+end
+
+end
+end

data/lib/hq/engine/engine.rb

@@ -1,20 +1,22 @@
+require "xml"
+
 require "hq/engine/libxmlruby-mixin"
+require "hq/transform/transformer"
 
 module HQ
 module Engine
+
 class Engine
 
 	include LibXmlRubyMixin
 
-	attr_accessor :main
+	attr_accessor :logger
+	attr_accessor :transform_backend
 
-	attr_accessor :results
+	attr_accessor :config_dir
+	attr_accessor :work_dir
 
-	def config_dir() main.config_dir end
-	def couch() main.couch end
-	def logger() main.logger end
-	def work_dir() main.work_dir end
-	def rule_provider() main.rule_provider end
+	attr_accessor :results
 
 	def schema_file() "#{work_dir}/schema.xml" end
 
@@ -87,8 +89,6 @@ class Engine
 
 			# process abstract config
 
-			require "hq/engine/transformer"
-
 			transformer = HQ::Engine::Transformer.new
 			transformer.parent = self
 
@@ -118,7 +118,6 @@ class Engine
 					item_doc.root
 				}
 
-
 			write_data_file schema_file, new_schemas
 
 			# restart if schema changed
@@ -185,9 +184,7 @@ class Engine
 
 		logger.notice "loading input from database"
 
-		require "xml"
-
-		logger.time "loading input from database" do
+		logger.time "converting input" do
 
 			@input_docs = {}
 			@input_strs = {}
@@ -197,38 +194,24 @@ class Engine
 
 			FileUtils.mkdir_p "#{work_dir}/input", :mode => 0700
 
-			rows = couch.view("root", "by_type")["rows"]
 			values_by_type = Hash.new
 
-			legacy = false
+			inputs = YAML.load File.read "#{work_dir}/input.yaml"
 
-			rows.each do |row|
+			inputs.each do
+				|input|
 
-				if legacy
+				id = input["id"]
+				type = input["type"]
+				value = input["value"]
 
-					type = row["value"]["mandar_type"]
-					value = row["value"]
-
-				else
-
-					type = row["value"]["type"]
-					value = row["value"]["value"]
-
-					row["id"] =~ /^current\/(.+)$/
-					value["_id"] = $1
-
-				end
+				value["_id"] = id
 
 				values_by_type[type] ||= Hash.new
-				values_by_type[type][value["_id"]] = value
-
-				main.continue
+				values_by_type[type][id] = value
 
 			end
 
-			change =
-				staged_change
-
 			schema =
 				load_schema_file "#{work_dir}/schema.xml"
 
@@ -251,20 +234,6 @@ class Engine
 				input_doc = XML::Document.new
 				input_doc.root = XML::Node.new "data"
 
-				if change
-					change["items"].each do |key, item|
-						case item["action"]
-						when "create", "update"
-							next unless key =~ /^#{Regexp.quote type}\//
-							values[key] = item["record"]
-						when "delete"
-							values.delete key
-						else
-							raise "Error"
-						end
-					end
-				end
-
 				sorted_values =
 					values.values.sort {
 						|a,b|
@@ -287,27 +256,6 @@ class Engine
 
 	end
 
-	def staged_change
-
-		return nil \
-			unless $deploy_mode == :staged
-
-		locks =
-			couch.get "mandar-locks"
-
-		return nil \
-			unless locks
-
-		change =
-			locks["changes"][$deploy_role]
-
-		return nil \
-			unless change
-
-		return change
-
-	end
-
 	def load_results
 
 		@results = {}
@@ -351,5 +299,6 @@ class Engine
 
 
 end
+
 end
 end

data/lib/hq/transform/ruby-backend.rb

@@ -0,0 +1,85 @@
+module HQ
+module Transform
+
+class RubyBackend
+
+  def session
+    return Session.new
+  end
+
+  def extensions
+    [ "rb" ]
+  end
+
+  class Session
+
+    def set_library_module name, content
+    end
+
+    def compile_xquery source, filename
+
+      @source = source
+      @filename = filename
+
+      @proc = Object.instance_eval source, filename
+
+    end
+
+    def run_xquery input, &callback
+
+      context = Context.new
+      context.callback = callback
+
+      @proc.call context
+
+      ret = XML::Document.new
+      ret.root = XML::Node.new "data"
+
+      context.results.each do
+        |node| ret.root << ret.import(node)
+      end
+
+      return ret.to_s
+
+    end
+
+    class Context
+
+      attr_accessor :callback
+      attr_accessor :results
+
+      def initialize
+        @results = []
+      end
+
+      def find type
+
+        strings =
+          @callback.call \
+            "search records",
+            {
+              "type" => type,
+            }
+
+        nodes =
+          strings.map {
+            |string|
+            XML::Document.string(string).root
+          }
+
+        return nodes
+
+      end
+
+      def write node
+        @results << node
+      end
+
+    end
+
+  end
+
+end
+
+end
+end

data/lib/hq/transform/transform-script.rb

@@ -0,0 +1,81 @@
+require "hq/logger"
+require "hq/tools/base-script"
+require "hq/tools/getopt"
+require "hq/transform/ruby-backend"
+require "hq/transform/transformer"
+
+module HQ
+module Transform
+
+class TransformScript < Tools::BaseScript
+
+  attr_accessor :logger
+  attr_accessor :transform_backend
+
+  def initialize
+
+    @logger = Logger.new
+    @logger.hostname = "local"
+    @logger.add_auto "debug"
+
+    @transform_backend = RubyBackend.new
+
+  end
+
+  def main
+    process_args
+    perform_transform
+  end
+
+  def process_args
+
+    @opts, @args =
+      Tools::Getopt.process @args, [
+
+        { :name => :input,
+          :required => true },
+
+        { :name => :rules,
+          :required => true },
+
+        { :name => :include,
+          :required => true },
+
+        { :name => :schema,
+          :required => true },
+
+        { :name => :output,
+          :required => true }
+
+      ]
+
+    @args.empty? \
+      or raise "Extra args on command line"
+
+  end
+
+  def perform_transform
+
+    transformer =
+      Engine::Transformer.new
+
+    transformer.parent = self
+
+    transformer.schema_file = @opts[:schema]
+    transformer.rules_dir = @opts[:rules]
+    transformer.include_dir = @opts[:include]
+    transformer.input_dir = @opts[:input]
+    transformer.output_dir = @opts[:output]
+
+    result =
+      transformer.rebuild
+
+    raise "Error" \
+      unless result[:success]
+
+  end
+
+end
+
+end
+end

data/lib/hq/{engine → transform}/transformer.rb

@@ -1,4 +1,5 @@
 require "hq/engine/libxmlruby-mixin"
+require "hq/engine/rule-error"
 
 module HQ
 module Engine
@@ -9,7 +10,7 @@ class Transformer
 	attr_accessor :parent
 
 	def logger() parent.logger end
-	def rule_provider() parent.rule_provider end
+	def transform_backend() parent.transform_backend end
 
 	attr_accessor :schema_file
 	attr_accessor :rules_dir
@@ -35,11 +36,18 @@ class Transformer
 		Dir.glob("#{rules_dir}/**/*").each do
 			|filename|
 
+			extensions_re =
+				"(?:%s)" % [
+					transform_backend.extensions
+						.map { |ext| Regexp.quote ext }
+						.join("|")
+				]
+
 			next unless filename =~ /^
 				#{Regexp.quote "#{rules_dir}/"}
 				(
 					(.+)
-					\. (xquery)
+					\. (#{extensions_re})
 				)
 			$/x
 
@@ -65,15 +73,15 @@ class Transformer
 
 	end
 
-	def init_rule_provider_session
+	def init_backend_session
 
-		@rule_provider_session =
-			rule_provider.session
+		@backend_session =
+			transform_backend.session
 
 		# add hq module
 		# TODO move this somewhere
 
-		@rule_provider_session.set_library_module \
+		@backend_session.set_library_module \
 			"hq",
 			"
 				module namespace hq = \"hq\";
@@ -105,7 +113,7 @@ class Transformer
 
 			remove_output
 
-			init_rule_provider_session
+			init_backend_session
 
 			@data = {}
 
@@ -186,7 +194,7 @@ class Transformer
 			path =~ /^ #{Regexp.quote include_dir} \/ (.+) $/x
 			name = $1
 
-			@rule_provider_session.set_library_module \
+			@backend_session.set_library_module \
 				name,
 				File.read(path)
 
@@ -318,12 +326,12 @@ class Transformer
 
 			begin
 
-				@rule_provider_session.compile_xquery \
+				@backend_session.compile_xquery \
 					rule[:source],
 					rule[:filename]
 
 				result_str =
-					@rule_provider_session.run_xquery \
+					@backend_session.run_xquery \
 						"<xml/>" \
 				do
 					|name, args|

data/spec/hq/{engine → transform}/transformer-spec.rb

@@ -1,4 +1,4 @@
-require "hq/engine/transformer"
+require "hq/transform/transformer"
 
 module HQ
 module Engine

metadata

@@ -1,15 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: hq-engine
 version: !ruby/object:Gem::Version
-  version: 0.0.13
+  version: 0.1.0
 platform: ruby
 authors:
 - James Pharaoh
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-01 00:00:00.000000000 Z
+date: 2013-05-20 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: hq-logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+- !ruby/object:Gem::Dependency
+  name: hq-tools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.8.1
 - !ruby/object:Gem::Dependency
   name: multi_json
   requirement: !ruby/object:Gem::Requirement
@@ -30,14 +58,42 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 1.2.1
+        version: 1.3.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+- !ruby/object:Gem::Dependency
+  name: hq-dev
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.18
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.18
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.7.7
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 1.2.1
+        version: 1.7.7
 - !ruby/object:Gem::Dependency
   name: libxml-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -58,56 +114,56 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 10.0.3
+        version: 10.0.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 10.0.3
+        version: 10.0.4
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 2.12.0
+        version: 2.13.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 2.12.0
+        version: 2.13.0
 - !ruby/object:Gem::Dependency
   name: rspec_junit_formatter
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.6
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.6
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.7.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.7.1
 description: HQ transformation engine
 email:
 - james@phsys.co.uk
@@ -115,6 +171,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- lib/hq/engine/engine-script.rb
 - lib/hq/engine/engine.rb
 - lib/hq/engine/register-engine-commands.rb
 - lib/hq/engine/subprocess-rule-provider/self.rb
@@ -122,12 +179,18 @@ files:
 - lib/hq/engine/subprocess-rule-provider/start.rb
 - lib/hq/engine/libxmlruby-mixin.rb
 - lib/hq/engine/unlock-command.rb
-- lib/hq/engine/transformer.rb
 - lib/hq/engine/rule-error.rb
+- lib/hq/transform/ruby-backend.rb
+- lib/hq/transform/transform-script.rb
+- lib/hq/transform/transformer.rb
+- features/engine/engine.feature
+- features/transform/transform.feature
+- features/support/steps.rb
+- features/support/env.rb
 - spec/hq/engine/libxmlruby-mixin-spec.rb
 - spec/hq/engine/subprocess-rule-provider/self-spec.rb
 - spec/hq/engine/subprocess-rule-provider/session-spec.rb
-- spec/hq/engine/transformer-spec.rb
+- spec/hq/transform/transformer-spec.rb
 homepage: https://github.com/jamespharaoh/hq-engine
 licenses: []
 metadata: {}
@@ -152,7 +215,11 @@ signing_key:
 specification_version: 4
 summary: HQ engine
 test_files:
+- features/engine/engine.feature
+- features/transform/transform.feature
+- features/support/steps.rb
+- features/support/env.rb
 - spec/hq/engine/libxmlruby-mixin-spec.rb
 - spec/hq/engine/subprocess-rule-provider/self-spec.rb
 - spec/hq/engine/subprocess-rule-provider/session-spec.rb
-- spec/hq/engine/transformer-spec.rb
+- spec/hq/transform/transformer-spec.rb