build-graph 0.3.6 → 0.3.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d39ed0195a0d9781587caca742e099f8d06d3387
-  data.tar.gz: bffbd32d48639c06da1ae2d02f41d54bee2d7b99
+  metadata.gz: 9e451eb353fae615423fa48f9f20a2890fcc6157
+  data.tar.gz: 78c36724ea7dc989d38c4b908fc9b9d5a36db7da
 SHA512:
-  metadata.gz: 4fcd3f3bbdf5e1ae87e3f5a7ad7ecec4c6bb27ee68b19bcb545acc656be1b740de901774f069927c3fa0e95418b43a919728e7579829412fdc0b22f1ae969205
-  data.tar.gz: d46d6eb6978817802cb270626cd8cc5f3bf36f7b7c2d65f6809f921ff4bcb6bdc8cefccc970baba730cf3c75f4c3350385b53035fc33d0e73282ecaac6aae486
+  metadata.gz: 80e50b90964b646412353ad187c787a4d40e47e4b11498bfe9f3c2df3e4c4f338b8f1b1b8909dc855204f3e8f26652735e3db9e6d6d8b706852dbeb3fbb7b5d4
+  data.tar.gz: 67c5ce3bdf104256332a94ddd12becb2516d95941a3d697bdf6d753d6b4262d64e33fab17487be0b570ebb8f6d51650ffc967fd2af9b451b46dcb3d105121015

data/README.md

@@ -22,7 +22,25 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+A build graph is an abstract set of `[input, process, output]` nodes. A node executes it's proces within the context of a `Task` which represents a specific set of inputs and outputs and is managed within a `Walker` that walks over graph nodes, regenerating tasks where required. If inputs or outputs change (i.e. become dirty), the task is destroyed and regenerated.
+
+A `Walker` is used to traverse the build graph once. As it walks over the graph it builds a set of `Edge` relationships between nodes and only traverses relationships which are complete `Walker#wait_on_paths`. Parent nodes also wait until all their children are complete `Walker#wait_on_nodes` It also keeps track of failures `Walker#failed?` and fails all dependencies of a node.
+
+A `Task` is instantiated once per node when traversing the graph. The task represents a specific process being applied to the graph, e.g. build, clean, etc. It is responsible for actually performing any real actions and providing the methods to do so. A `Task` contains all details about the specific graph state at that point in time, e.g. `Task#children` and updating the node state in `Task#exit`. Statistics on the build graph are also captured through `Task` and `Walker`, e.g. number of nodes visited, etc.
+
+### Inputs and Outputs
+
+Inputs to a node should be all on-disk state and any additional parameters which cause it's behavior to produce different results.
+
+Outputs from a node should be all files that are generated directly by the processes within the node and sometimes it's children.
+
+### Dirty Propagation
+
+A `Node` has a set of `#inputs` and `#outputs` but these are abstract. A `Task`, at the time of execution, captures it's inputs and outputs and these may be monitored for changes in real time. The simplest way to cause a task to regenerate is to simply remove it from the existing graph and it will be regenerated.
+
+File changes are currently detected using `File::mtime` as this is generally a good trade off between efficiency and accuracy.
+
+When a task is marked as dirty, it also marks all it's outputs as being dirty, which in cause could mark other tasks as dirty. This is the mechanism for which dirtiness propagates through the graph. The walker should only have to traverse the graph once to build it completely. If multiple updates are required (i.e. buidling one part of the graph implicitly dirties another part of the graph), the specification of the graph is incomplete and this may lead to problems within the build graph.
 
 ## Contributing
 

data/build-graph.gemspec

@@ -23,13 +23,13 @@ Gem::Specification.new do |spec|
 	spec.required_ruby_version = '>= 2.0'
 	
 	spec.add_dependency "process-group", "~> 0.2.1"
-	spec.add_dependency "build-files", "~> 0.3.0"
+	spec.add_dependency "build-files", "~> 0.3.3"
 	
 	spec.add_dependency "system"
 	spec.add_dependency "rainbow", "~> 2.0.0"
 	
 	spec.add_development_dependency "bundler", "~> 1.3"
 	spec.add_development_dependency "rspec", "~> 3.0.0"
-	spec.add_development_dependency "build-makefile", "~> 0.2.0"
+	spec.add_development_dependency "build-makefile", "~> 0.3.0"
 	spec.add_development_dependency "rake"
 end

data/lib/build/graph/controller.rb

@@ -28,12 +28,12 @@ require_relative 'edge'
 module Build
 	module Graph
 		# The top level graph controller is responsible for managing build graph state.
-		class Controller < Files::Monitor
+		class Controller
 			def initialize
 				super
-			
+				
 				@nodes = {}
-			
+				
 				build_graph!
 			end
 			
@@ -58,7 +58,7 @@ module Build
 				
 				walker = walk do |walker, node|
 					nodes << node
-				
+					
 					yield walker, node
 				end
 				
@@ -97,6 +97,11 @@ module Build
 				
 				return walker
 			end
+			
+			# What to do when a task has a trasient failure:
+			def task_failure(error, task)
+				$stderr.puts Rainbow("Error: #{error.inspect}").red
+			end
 		end
 	end
 end

data/lib/build/graph/edge.rb

@@ -31,45 +31,60 @@ module Build
 				
 				# The number of inputs we are waiting for:
 				@count = count
+				@vertices = 0
 				
 				@failed = []
 			end
-		
+			
 			attr :failed
-		
+			
 			attr :fiber
 			attr :count
 			
-			# Wait until all inputs to the edge have been traversed.
+			# Wait until all inputs to the edge have been traversed. Returns false if failed?
 			def wait
 				if @count > 0
 					Fiber.yield
 				end
-			
-				failed?
+				
+				succeeded?
 			end
-		
+			
 			attr :failed
-		
+			
 			def failed?
 				@failed.size != 0
 			end
 			
+			def succeeded?
+				@failed.size == 0
+			end
+			
 			# Traverse the edge, mark the edge as failed if the source was also failed.
-			def traverse(node)
+			def traverse(task)
 				@count -= 1
-			
-				if node.failed?
-					@failed << node
+				
+				# The entire edge fails if any individual task fails.
+				if task.failed?
+					@failed << task
 				end
-			
+				
 				if @count == 0
 					@fiber.resume
 				end
 			end
 			
+			def skip!(task)
+				@vertices += 1
+				
+				if task.failed?
+					@failed << task
+				end
+			end
+			
 			# Increase the number of traversals we are waiting for.
 			def increment!
+				@vertices += 1
 				@count += 1
 			end
 		end

data/lib/build/graph/node.rb

@@ -19,140 +19,69 @@
 # THE SOFTWARE.
 
 require 'build/files/state'
+require 'build/files'
+
+require 'rainbow'
 
 module Build
 	module Graph
+		# This is essentialy a immutable key:
 		class Node
-			def initialize(controller, inputs, outputs)
-				@controller = controller
-			
-				@state = Files::IOState.new(inputs, outputs)
-			
-				@status = :unknown
-				@fiber = nil
-			
+			def initialize(inputs, outputs, process)
 				# These are immutable - rather than change them, create a new node:
 				@inputs = inputs
 				@outputs = outputs
-			
-				@controller.add(self)
-			end
-		
-			def eql?(other)
-				other.kind_of?(self.class) and @inputs.eql?(other.inputs) and @outputs.eql?(other.outputs)
-			end
-		
-			def hash
-				[@inputs, @outputs].hash
-			end
-		
-			def directories
-				@state.files.roots
-			end
-		
-			def remove!
-				@controller.delete(self)
-			end
-		
-			# It is possible this function is called unnecessarily. The state check confirms whether a change occurred or not.
-			def changed!(outputs = [])
-				# Don't do anything if we are already dirty.
-				return if dirty?
-		
-				if @state.intersects?(outputs) || @state.update!
-					# puts "** Dirty: #{@inputs.to_a.inspect} -> #{@outputs.to_a.inspect}"
-			
-					# Could possibly use unknown status here.
-					@status = :dirty
-			
-					# If this node changes, we force all other nodes which depend on this node to be dirty.
-					@controller.update(directories, @outputs)
-				end
+				
+				# Represents an abstract process, e.g. a name or a function.
+				@process = process
 			end
 			
 			attr :inputs
 			attr :outputs
+			attr :process
 			
-			# The IOState for this node.
-			attr :state
+			# Nodes that inherit outputs are special in the sense that outputs are not available until all child nodes have been evaluated.
+			def inherit_outputs?
+				@outputs == :inherit
+			end
 			
-			# The status of this node.
-			attr :status
-		
-			def unknown?
-				@status == :unknown
+			# This computes the most recent modified time for all inputs.
+			def modified_time
+				modified_time = @inputs.map{|path| path.modified_time}.max
 			end
-		
+			
+			# This is a canonical dirty function. All outputs must exist and must be newer than all inputs. This function is not efficient, in the sense that it must query all files on disk for last modified time.
 			def dirty?
-				@status == :dirty
-			end
-		
-			def clean?
-				@status == :clean
-			end
-		
-			def clean!
-				@status = :clean
-			end
-		
-			def fail!
-				@status = :failed
-			end
-		
-			def failed?
-				@status == :failed
+				if inherit_outputs?
+					return true
+				else
+					# Dirty if any outputs don't exist:
+					return true if @outputs.any?{|path| !path.exist?}
+					
+					# Dirty if input modified after any output:
+					input_modified_time = self.modified_time
+					
+					# Outputs should always be more recent than their inputs:
+					return true if @outputs.any?{|output_path| output_path.modified_time < input_modified_time}
+				end
+				
+				return false
 			end
-		
-			def updating?
-				@fiber != nil
+			
+			def eql?(other)
+				other.kind_of?(self.class) and @inputs.eql?(other.inputs) and @outputs.eql?(other.outputs) and @process.eql?(other.process)
 			end
-		
-			# If we are in the initial state, we need to check if the outputs are fresh.
-			def update_status!
-				#puts "Update status: #{@inputs.inspect} -> #{@outputs.inspect} (status=#{@status} @fiber=#{@fiber.inspect}) @status=#{@status} @state.fresh?=#{@state.fresh?}"
 			
-				if @status == :unknown
-					# This could be improved - only stale files should be reported, instead we report all.
-					unless @state.fresh?
-						changed!(self.inputs)
-					else
-						@status = :clean
-					end
-				end
+			def hash
+				[@inputs, @outputs, @process].hash
 			end
-		
+			
 			def inspect
-				"<#{dirty? ? '*' : ''}inputs=#{inputs.inspect} outputs=#{outputs.inspect} fiber=#{@fiber.inspect} fresh=#{@state.fresh?}>"
-			end
-		
-			def requires_update?
-				not clean?
+				"<#{self.class.name} #{@inputs.inspect} => #{@outputs.inspect} by #{@process.inspect}>"
 			end
-		
-			# Perform some actions to update this node, returns when completed, and the node is no longer dirty.
-			def update!(walker)
-				#puts "Walking #{@inputs.to_a.inspect} -> #{@outputs.to_a.inspect} (dirty=#{dirty?} @fiber=#{@fiber.inspect})"
 			
-				# If a fiber already exists, this node is in the process of updating.
-				if requires_update? and @fiber == nil
-					# puts "Beginning: #{@inputs.to_a.inspect} -> #{@outputs.to_a.inspect}"
-				
-					@fiber = Fiber.new do
-						task = walker.task(self)
-					
-						task.visit
-					
-						# Commit changes:
-						# puts "** Committing: #{@inputs.to_a.inspect} -> #{@outputs.to_a.inspect}"
-					
-						@state.update!
-						@fiber = nil
-					
-						task.exit
-					end
-			
-					@fiber.resume
-				end
+			def self.top(inputs = Files::Paths::NONE, outputs = :inherit, &block)
+				self.new(inputs, outputs, block)
 			end
 		end
 	end

data/lib/build/graph/task.rb

@@ -0,0 +1,168 @@
+# Copyright, 2014, by Samuel G. D. Williams. <http://www.codeotaku.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+module Build
+	module Graph
+		class Task
+			def initialize(walker, node)
+				@walker = walker
+				
+				@walker.tasks[node] = self
+				
+				@node = node
+				
+				# If the execution of the node fails, this is where we save the error:
+				@error = nil
+				
+				@children = []
+				
+				@state = nil
+				
+				@inputs_failed = false
+			end
+			
+			attr :inputs
+			attr :outputs
+			
+			attr :children
+			attr :state
+			
+			attr :walker
+			
+			attr :node
+			
+			# A list of any inputs whose relevant tasks failed:
+			attr :inputs_failed
+			
+			# Derived task should override this function to provide appropriate behaviour.
+			def visit
+				update_inputs_and_outputs
+				
+				# Inforn the walker a new task is being generated for this node:
+				@walker.enter(self)
+				
+				@fiber = Fiber.new do
+					# If all inputs were good, we can update the node.
+					if wait_for_inputs?
+						begin
+							yield
+						rescue TransientError => error
+							fail!(error)
+						end
+					else
+						fail!(:inputs)
+					end
+					
+					unless wait_for_children?
+						fail!(:children)
+					end
+					
+					update_outputs
+					
+					@state ||= :complete
+					
+					@walker.exit(self)
+				end
+				
+				# Schedule the work, hopefully synchronously:
+				@fiber.resume
+				
+				# This allows the child task to be passed back to the parent when it is first invoked.
+				return self
+			end
+			
+			def invoke(node)
+				child_task = @walker.call(node)
+				
+				raise ArgumentError.new("Invalid child task") unless child_task
+				
+				@children << child_task
+			end
+			
+			def failed?
+				@state == :failed
+			end
+			
+			def complete?
+				@state == :complete
+			end
+			
+			# Returns true if the outputs of the task are out of date w.r.t. the inputs.
+			# Currently, does not take into account if the input is a glob and files have been added.
+			def dirty?
+				@outputs.dirty?(@inputs)
+			end
+			
+			def changed!
+				@walker.delete(@node)
+			end
+			
+			def directories
+				@inputs.roots + @outputs.roots
+			end
+			
+			def inspect
+				"<#{self.class}:#{'0x%X' % self.object_id} #{@node.inspect} #{@state}>"
+			end
+			
+			attr :error
+			attr :state
+			
+		protected
+			def update_inputs_and_outputs
+				# If @node.inputs is a glob, this part of the process converts the glob into an actual list of files.
+				@inputs = Files::State.new(@node.inputs)
+				
+				unless @node.inherit_outputs?
+					@outputs = Files::State.new(@node.outputs)
+				end
+			end
+			
+			def children_outputs
+				@children.collect(&:outputs).inject(Files::Paths::NONE, &:+)
+			end
+			
+			def update_outputs
+				if @node.inherit_outputs?
+					@outputs = Files::State.new(self.children_outputs)
+				else
+					# After the task has finished, we update the output states:
+					@outputs.update!
+				end
+			end
+			
+			def fail!(error)
+				@error = error
+				@state = :failed
+			end
+			
+			# Returns false if any input failed.
+			def wait_for_inputs?
+				# Wait on any inputs, returns whether any inputs failed:
+				@walker.wait_on_paths(@inputs)
+			end
+			
+			# Returns false if any child failed.
+			def wait_for_children?
+				@walker.wait_for_children(self, @children)
+			end
+		end
+	end
+end