grably 0.0.1

data/lib/grably/core/task/expand.rb

@@ -0,0 +1,15 @@
+module Grably
+  module Core
+    module TaskExtensions
+      # Add expand method to Task
+      module Expand
+        # Expands expression in task context
+        # @param expr [Object] expand expression
+        # @return Array<Product> result of expand in task context
+        def expand(expr)
+          Product.expand(expr, self)
+        end
+      end
+    end
+  end
+end

data/lib/grably/core/task/jobs.rb

@@ -0,0 +1,58 @@
+module Grably
+  module Core
+    module TaskExtensions
+      # # Jobs
+      # @see Grably::Core::Job
+      module Jobs
+        def method_missing(meth, *args, &block)
+          job_class = find_job_class(meth.to_s)
+          if job_class
+            execute_job_with_args(args, job_class, meth)
+          else
+            super
+          end
+        end
+
+        def execute_job_with_args(args, job_class, meth)
+          working_dir = job_dir(task_dir, meth.to_s)
+          FileUtils.mkdir_p(working_dir)
+          job_class.new.run(self, working_dir, *args)
+        end
+
+        # Create working directory for instantiated job inside task directory
+        # @param [String] base_dir task working directory
+        # @param [String] job_name Grably::Job call name
+        # @return [String] job working directory
+        def job_dir(base_dir, job_name)
+          # All this flow is working under assumption that all task jobs called
+          # in same order. We store counter for each job in Task instance and
+          # it updated throug all task live time. Each time task instance is
+          # recreated we use frech (zero) counter
+          counter = (jobs[job_name] || -1) + 1
+          jobs[job_name] = counter
+          name = [job_name, counter.to_s.rjust(3, '0')].join('-')
+          File.join(base_dir, name)
+        end
+
+        def jobs
+          @jobs ||= {}
+        end
+
+        def respond_to_missing?(meth, include_private = false)
+          find_job_class(meth) || super
+        end
+
+        private
+
+        def find_job_class(name)
+          n = name.to_sym
+          all_classes = Grably::Job.jobs.flat_map do |c|
+            ObjectSpace.each_object(Class).select { |klass| klass <= c }
+          end
+
+          all_classes.find { |c| c.job_call_name == n }
+        end
+      end
+    end
+  end
+end

data/lib/grably/job.rb

@@ -0,0 +1,28 @@
+require_relative 'core/product'
+require_relative 'core/digest'
+
+require_relative 'job/exceptions'
+require_relative 'job/manifest'
+require_relative 'job/class'
+require_relative 'job/instance'
+
+module Grably
+  # TBD
+  module Job
+    # includes "Job::InstanceMethods" and extends "Job::ClassMethods"
+    # using the Job.included callback.
+    # @!parse include Job::InstanceMethods
+    # @!parse extend Job::ClassMethods
+    class << self
+      def included(receiver)
+        receiver.extend         ClassMethods
+        receiver.send :include, InstanceMethods
+        jobs << receiver
+      end
+
+      def jobs
+        @jobs ||= []
+      end
+    end
+  end
+end

data/lib/grably/job/class.rb

@@ -0,0 +1,93 @@
+module Grably
+  module Job
+    module ClassMethods # :nodoc:
+      # Mark instance variable as source product container. When field contains
+      # source product job will track its state between launches. If product
+      # was changed since last run job is considered as changed and will be
+      # rebuild. This method also generates attribute_reader for given variable.
+      # @param name [Symbol] attribute name
+      # @param extras [Hash] extra arguments. Now unsued.
+      def src(name, extras = {})
+        class_exec do
+          attr_reader name
+          register_job_argument(name, :src, extras)
+        end
+      end
+
+      # Mark instance variable as multiple source product container. When field
+      # contains multiple src products job will track each product state
+      # between launches. If any of source products was changed since last run
+      # job is considered as changed and will be rebuild. This method also
+      # generates attribute_reader for given variable.
+      # @param name [Symbol] attribute name
+      # @param extras [Hash] extra arguments. Now unsued.
+      def srcs(name, extras = {})
+        class_exec do
+          attr_reader name
+          register_job_argument(name, :srcs, extras)
+        end
+      end
+
+      # Mark instance variable as incremental sources container. When field
+      # contains incremental sources it does not impact on job state. But job
+      # setup will generate state delta between two launches. This allows to
+      # decide if job should be rebuild completely or only changed files should
+      # be rebuild. This method generates two syntatic instance methods:
+      # * standart attribute accessor which contains current products
+      # * ! attribute_accessor (i.e. foo! ) which will return delta array in
+      #   following format: [ modifications, additions, deletions ]
+      # @param name [Symbol] attribute name
+      # @param extras [Hash] extra arguments. Now unsued.
+      def srcs!(name, extras = {})
+        class_exec do
+          attr_reader name
+          register_job_argument(name, :isrcs, extras)
+          eval("def #{name}!; @deltas[:#{name}]; end") # rubocop:disable Security/Eval
+        end
+      end
+
+      # Mark instance variable as option container. When field contains option
+      # job will track its value between launches. If value was changed since
+      # last run job is considered as changed and will be rebuild. This method
+      # also generates attribute_reader for given variable.
+      # @param name [Symbol] attribute name
+      # @param extras [Hash] extra arguments. Now unsued.
+      def opt(name, extras = {})
+        class_exec do
+          attr_reader name
+          register_job_argument(name, :opt, extras)
+        end
+      end
+
+      def register_job_argument(name, type, extras)
+        job_args[name] = [type, extras]
+      end
+
+      def job_args
+        @job_args ||= {}
+      end
+
+      def job_call_name
+        return unless name # TODO: in runtime some anonymous sublcass instances
+        # can be created find out when and why
+        # job_call_name by user with call_as method. Yet it is not required
+        # if job_call_name empty we'll try to sytetize job name
+        @job_call_name ||= synthesize_job_name
+      end
+
+      def call_as(name)
+        @job_call_name = name
+      end
+
+      # Try to syntetize job name from class name
+      # * Strip all parent modules names
+      # * Cut off Job postfix from class name if any
+      # * convert class name to snake case
+      def synthesize_job_name
+        klass_name = name.scan(/[^:]+/).last
+        klass_name = klass_name[/(.+)Job/, 1] || klass_name
+        klass_name.scan(/[A-Z][a-z]+/).map(&:downcase).join('_').to_sym
+      end
+    end
+  end
+end

data/lib/grably/job/instance.rb

@@ -0,0 +1,159 @@
+module Grably
+  module Job
+    module InstanceMethods # rubocop:disable  Metrics/ModuleLength, Style/Documentation
+      def run(task, working_dir, *args) # rubocop:disable  Metrics/MethodLength,  Metrics/AbcSize
+        prepare(task, working_dir)
+        initialize_state(*args) if stateful?
+
+        if changed?
+          clean unless incremental?
+          log "  * [#{self.class.job_call_name}] building"
+          result = @manifest.result = Grably::Core::Product.expand(build)
+          @manifest.dump
+        else
+          log "  * [#{self.class.job_call_name}] uptodate"
+          result = @manifest.result
+        end
+
+        result
+      rescue StandardError => err
+        # If any error occured this will force rebuild on next run
+        @manifest.remove if @manifest
+        raise(err)
+      end
+
+      # When job is changed it need to be reevaluated so build method will be
+      # launched. If job remains unchanged previous result will be returned
+      def changed?
+        @changed
+      end
+
+      # Means that job has state which can be tracked between launches. If
+      # job is stateless it always changed.
+      def stateful?
+        @stateful
+      end
+
+      # If job contains incremental srcs it is incremental. Incremental job
+      # does not clean self state between launches. It should be managed by
+      # user.
+      def incremental?
+        @job_args.any? { |_name, desc| desc.first == :isrc }
+      end
+
+      def job_dir(path = nil)
+        path.nil? ? @job_dir : File.join(@job_dir, path)
+      end
+
+      def meta
+        @manifest.meta
+      end
+
+      # Celans job state by removing all files from working directory
+      def clean
+        FileUtils.rm_rf(Dir[File.join(@job_dir, '*')])
+      end
+
+      private
+
+      def prepare(task, working_dir)
+        @job_dir = working_dir # initialize job dir
+        @t = task
+        @job_args = job_args_lookup
+        # We need to track state only if job has arguments.
+        @stateful = !@job_args.empty?
+        # If job is stateful it is not changed by default. Only after stat will
+        # be initialized we can say if it changed. If job has no state we should
+        # always rebuild it. So changed != stateful
+        @changed = !@stateful
+        # In case if job has incremental sources we should keep deltas
+        # somewhere. Deltas are kept in hash where they stored by field name.
+        @deltas = {}
+      end
+
+      # Executes job state initialization by assigning values to instance
+      # variables
+      def initialize_state(*args)
+        # Load previous state
+        @manifest = Manifest.new(job_dir)
+        _loaded = @manifest.load # try to load manifest
+
+        if self.class.method_defined?(:setup)
+          # This means that class has user defined method for initialization so
+          # we should call this method for setup
+          setup(*args)
+        else
+          # If no setup method defined we will use synthetic setup which assumes
+          # that arguments is a Hash where keys is job argument names
+          synthetic_setup(*args)
+        end
+        expand_job_arguments
+      end
+
+      def synthetic_setup(args)
+        check_synthetic_arguments(args)
+        args.each { |k, v| instance_variable_set("@#{k}", v) }
+      end
+
+      def check_synthetic_arguments(args)
+        raise "Expected Hash got #{args.inspect}" unless args.is_a?(Hash)
+        job_args = @job_args.keys
+        extra = args.keys - job_args
+        raise "Unknown arguments: #{extra.join(', ')}" unless extra.empty?
+        missing = job_args - args.keys
+        raise "Missing arguments #{missing.join(', ')}" unless missing.empty?
+      end
+
+      # Walks through all job arguments and does proper expand operation for its
+      # value
+      def expand_job_arguments
+        @job_args.each do |name, desc|
+          type, _extras = desc
+          value = instance_variable_get("@#{name}")
+          update_argument(name, type, value)
+        end
+      end
+
+      def update_argument(name, type, value) # rubocop:disable Metrics/MethodLength
+        case type
+        when :src
+          # src expects single product
+          value = Product.expand(value, @t)
+          raise(ArgumentError, "Expected only one product for #{name}") unless value.length == 1
+          value = value.first
+        when :srcs, :isrcs
+          # src and isrcs expects multiple products
+          value = Product.expand(value, @t)
+        when :opt # rubocop:disable Lint/EmptyWhen
+          # do nothing, we do not expand opt values. keeping them as is
+        when nil
+          raise(ArgumentError, "#{name} not defined")
+        else
+          raise(ArgumentError, "Unknown type #{type} for #{name}")
+        end
+
+        @manifest.update(name, type, value, ->(*a) { on_job_arg_set(*a) })
+        instance_variable_set("@#{name}", value)
+      end
+
+      # rubocop:disable Metrics/ParameterLists
+      def on_job_arg_set(name, type, old_digest, old_val, new_digest, new_val)
+        if type == :isrcs
+          # isrcs never impact changed state. We just need to gather infomration
+          # about what actualy was changed
+          @deltas[name] = Core::Digest.diff_digests(old_digest, new_digest)
+        elsif old_val != new_val || old_digest != new_digest
+          @changed = true
+        end
+      end
+      # rubocop:enable Metrics/ParameterLists
+
+      def job_args_lookup(klass = self.class, args = {})
+        job_args_lookup(klass.superclass, args) if klass.superclass
+        args.update(klass.job_args) if klass.included_modules.include?(Grably::Job)
+
+        args
+      end
+    end
+  end
+end

data/lib/grably/job/manifest.rb

@@ -0,0 +1,67 @@
+require 'ostruct'
+require 'fileutils'
+
+module Grably
+  module Job
+    # Manifest file with job data. Manifest keeps information about previous run
+    # including passed arguments and output file digests
+    class Manifest
+      # Manifest file location relative to jobdir
+      MANIFEST = '.manifest'.freeze
+
+      attr_reader :manifest_file
+
+      def initialize(job_dir)
+        @job_dir = job_dir
+        @data = OpenStruct.new(src: {}, srcs: {}, opt: {}, result: nil, meta: {})
+        @manifest_file = File.join(job_dir, MANIFEST)
+      end
+
+      def update(name, type, value, update_hook) # rubocop:disable Metrics/MethodLength,  Metrics/AbcSize
+        case type
+        when :src
+          old_value, old_digest = @data.src[name]
+          new_digest = Grably::Digest.digest(value).first
+        when :srcs
+          old_value, old_digest = @data.srcs[name]
+          new_digest = Grably::Digest.digest(*value)
+        when :opt
+          old_value, _old_digest = @data.opt[name]
+          old_digest = nil
+          new_digest = nil
+        else
+          raise ArgumentError, 'Invalid type: ' + type
+        end
+
+        @data.send(type)[name] = [value, new_digest]
+        update_hook.call(name, type, old_value, old_digest, value, new_digest)
+        [old_value, old_digest]
+      end
+
+      def result=(products)
+        digests = Grably::Digest.digest(*products)
+        @data.result = [products, digests]
+      end
+
+      def meta
+        @data.meta
+      end
+
+      def result
+        @data.result.first
+      end
+
+      def load
+        @data = load_obj(manifest_file) if File.exist?(manifest_file)
+      end
+
+      def dump
+        save_obj(manifest_file, @data)
+      end
+
+      def remove
+        FileUtils.rm_f(manifest_file)
+      end
+    end
+  end
+end

data/lib/grably/jobs.rb

@@ -0,0 +1,4 @@
+require_relative 'jobs/text'
+require_relative 'jobs/zip'
+require_relative 'jobs/sync'
+require_relative 'jobs/upload'

data/lib/grably/jobs/sync.rb

@@ -0,0 +1,91 @@
+module Grably # :nodoc:
+  # TBD
+  class SyncJob
+    include Grably::Job
+
+    PROTO_SSH = %r{^ssh://(.*)$}
+    SSH_HOST = /(.+?):(.+?)@(.+)/
+    DEFAULT_RSYNC_PARAMS = %w(-avz --progress --delete).freeze
+
+    srcs :files
+    opt :dst
+
+    opt :host
+    opt :proto
+    opt :no_partial
+    opt :ssh_key
+    opt :ssh_pass
+    opt :ssh_port
+
+    def setup(srcs, dst = nil, _p = {})
+      @files = srcs
+      @dst = dst || job_dir
+
+      @proto = :file
+
+      unpack_dst_hash(@dst) unless @dst.is_a?(String)
+      @dst.match(PROTO_SSH) do |m|
+        @proto = :ssh
+        @dst = m[1]
+      end
+      setup_ssh_opts if @proto == :ssh
+    end
+
+    def changed?
+      true
+    end
+
+    def build
+      trace "Syncing products to #{dst}"
+      cp_smart(files, dst, log: proto == :file)
+      ssh_sync if proto == :ssh
+      proto == :ssh ? [] : dst
+    end
+
+    private
+
+    def ssh_cmd
+      cmd = %w(ssh)
+      cmd += ['-i', ssh_key] if ssh_key
+      cmd += ['-p', ssh_port] if ssh_port
+      cmd =  ['sshpass', '-p', ssh_pass] + cmd if ssh_pass
+      "'#{cmd.join(' ')}'"
+    end
+
+    def unpack_dst_hash(dst)
+      @proto = :ssh
+      @no_partial = dst[:no_partial]
+      @ssh_key = dst[:ssh_key]
+      @ssh_pass = dst[:ssh_pass]
+      @ssh_port = dst[:ssh_port]
+      @dst = dst[:host]
+    end
+
+    def ssh_sync
+      if files.empty?
+        warn 'Nothing to sync' if files.empty?
+      end
+
+      log "Syncing #{files.size} files to #{host}"
+      ['rsync', *rsync_params, @dst, '-e', ssh_cmd, host].run_log
+    end
+
+    def rsync_params
+      params = DEFAULT_RSYNC_PARAMS.dup
+      params << '--partial' unless no_partial
+
+      params
+    end
+
+    def setup_ssh_opts
+      @host = @dst
+      # '/' at the end will cause to sync internal file structure
+      @dst = job_dir('files') + '/'
+
+      @host.match(SSH_HOST) do |m|
+        @ssh_pass = m[2]
+        @host = [m[1], m[2]].join('@')
+      end
+    end
+  end
+end