wukong-deploy 0.1.1 → 0.2.0

data/.gitignore

@@ -57,3 +57,4 @@ away
 .rbx
 Gemfile.lock
 Backup*of*.numbers
+*.gem

data/Gemfile

@@ -1,4 +1,4 @@
-source :rubygems
+source 'https://rubygems.org'
 
 gemspec
 

data/lib/wukong-deploy.rb

@@ -26,8 +26,15 @@ module Wukong
         settings.define(:skip,    :description => "Skip existing files", :type => :boolean, :default => false)
         settings.define(:force,   :description => "Overwrite existing files", :type => :boolean, :default => false)
       when 'wu-hadoop'
-        require_relative('wukong-deploy/extensions')
+        require_relative('wukong-deploy/hadoop_extensions')
         settings[:command_prefix] = 'bundle exec'
+      when 'wu-storm'
+        require_relative('wukong-deploy/storm_extensions')
+        settings[:command_prefix] = 'bundle exec'
+      when 'wu-dump'
+        require_relative('wukong-deploy/dump_extensions')
+      when 'wu-sync', 'wu-sync-all'
+        require_relative('wukong-deploy/sync_extensions')
       end
     end
 

data/lib/wukong-deploy/deploy_pack.rb

@@ -1,5 +1,10 @@
 module Wukong
   module Deploy
+
+    # Return the name of this deploy pack.
+    def self.name
+      settings[:application]
+    end
     
     # Return the root directory of this deploy pack.
     #
@@ -58,11 +63,11 @@ module Wukong
       @root
     end
 
-    # The directory this deploy pack keeps local, sample data in.
+    # The directory this deploy pack puts all its application code in.
     #
     # @return [Pathname]
-    def self.data_dir
-      root.join('data')
+    def self.app_dir
+      root.join('app')
     end
 
     # The directory this deploy pack uses for configuration files.
@@ -71,12 +76,19 @@ module Wukong
     def self.config_dir
       root.join('config')
     end
+    
+    # The directory this deploy pack keeps local, sample data in.
+    #
+    # @return [Pathname]
+    def self.data_dir
+      root.join('data')
+    end
 
-    # The directory this deploy pack uses for temporary files.
+    # The directory this deploy pack uses for lib files.
     #
     # @return [Pathname]
-    def self.tmp_dir
-      root.join('tmp')
+    def self.lib_dir
+      root.join('lib')
     end
 
     # The directory this deploy pack uses for logs.
@@ -86,11 +98,33 @@ module Wukong
       root.join('log')
     end
 
-    # The diretory this deploy pack puts all its application code in.
+    # The directory this deploy pack puts all its scripts in.
     #
     # @return [Pathname]
-    def self.app_dir
-      root.join('app')
+    def self.script_dir
+      root.join('script')
+    end
+    
+    # The directory this deploy pack puts all its Ruby unit tests.
+    #
+    # @return [Pathname]
+    def self.spec_dir
+      root.join('spec')
+    end
+
+    # The directory this deploy pack puts all its non-Ruby source
+    # code.
+    #
+    # @return [Pathname]
+    def self.src_dir
+      root.join('src')
+    end
+    
+    # The directory this deploy pack uses for temporary files.
+    #
+    # @return [Pathname]
+    def self.tmp_dir
+      root.join('tmp')
     end
     
     private
@@ -148,9 +182,16 @@ module Wukong
     end
 
     # Read remote settings.
-    #
-    # FIXME -- not implemented yet.
     def self.read_remote_settings
+      return unless settings[:vcd]
+      if name.nil? || name.empty?
+        Wukong::Log.warn("Define the :application settings to give this deploy pack a name and be be able to read remote settings.")
+        return
+      end
+      topic = "deploy_packs.#{name}"
+      remote_settings = vayacondios_client.get(topic)
+      settings.merge(remote_settings) if remote_settings.success?
     end
+    
   end
 end

data/lib/wukong-deploy/dump_extensions.rb

@@ -0,0 +1,6 @@
+require_relative('extensions/uses_lockfile')
+require_relative('extensions/uses_file_state')
+Wukong::Load::DirectoryDumper.class_eval do
+  include Wukong::Deploy::UsesLockfileOverride
+  include Wukong::Deploy::UsesFileStateOverride
+end

data/lib/wukong-deploy/extensions/ftp_syncer.rb

@@ -0,0 +1,67 @@
+module Wukong
+  module Deploy
+
+    # Attaches to the hooks provided by the Wukong::Load::Syncer class
+    # to write data to Vayacondios.
+    module FTPSyncerOverride
+
+      # Saves the syncer as a stash in Vayacondios.
+      def before_sync
+        super()
+        Wukong::Deploy.vayacondios_client.set(vayacondios_topic, 'ftp', self)
+      end
+
+      # Announces a successful sync and updates the last sync state
+      # and time.
+      def after_sync
+        super()
+        Wukong::Deploy.vayacondios_client.announce(vayacondios_topic, {
+          success: true,
+          step:    'ftp',
+          files:   self.files,
+        }.tap { |e| e[:duration] = duration if duration })
+        now = Time.now.utc.to_i
+        last_ftp_sync_summary = { state: 1, time: now}
+        last_ftp_sync_summary[:data] = now unless self.files.empty?
+        Wukong::Deploy.vayacondios_client.set(vayacondios_topic, "ftp.last", last_ftp_sync_summary)
+      end
+
+      # Announces an error during a sync and updates the last sync
+      # state and time.
+      def on_error error
+        super(error)
+        Wukong::Deploy.vayacondios_client.announce(vayacondios_topic, {
+          success: false,
+          step:    'ftp',
+          error:   error.class,
+          message: error.message,
+          files:   self.files,
+        })
+        Wukong::Deploy.vayacondios_client.set(vayacondios_topic, "ftp.last", { state: 0, time: Time.now.utc.to_i })
+      end
+      
+      # Returns the Vayacondios topic for this FTP syncer.
+      #
+      # @return [String] the Vayacondios topic
+      def vayacondios_topic
+        "listeners.#{name}"
+      end
+
+      # Returns a representation of this FTP syncer suitable for a
+      # Vayacondios stash.
+      #
+      # @return [Hash]
+      def to_vayacondios
+        {
+          protocol: settings[:protocol],
+          username: settings[:username],
+          password: (settings[:password] ? true : false),
+          host:     settings[:host],
+          port:     port,
+          path:     settings[:path],
+        }
+      end
+
+    end
+  end
+end

data/lib/wukong-deploy/extensions/prepare_syncer.rb

@@ -0,0 +1,105 @@
+module Wukong
+  module Deploy
+
+    # Attaches to the hooks provided by the Wukong::Load::Syncer class
+    # to write data to Vayacondios.
+    module PrepareSyncerOverride
+
+      # Saves the syncer as a stash in Vayacondios.
+      def before_sync
+        super()
+        Wukong::Deploy.vayacondios_client.set(vayacondios_topic, 'prepare', self)
+      end
+
+      # Announces a successful sync and updates the last sync state
+      # and time.
+      def after_sync
+        super()
+        Wukong::Deploy.vayacondios_client.announce(vayacondios_topic, {
+          success: success?,
+          step:    'prepare',
+          counts:   counts,
+          files:    files,
+        }.tap { |e| e[:duration] = duration if duration })
+        Wukong::Deploy.vayacondios_client.set(vayacondios_topic, "prepare.last", { state: (success? ? 1 : 0), time: Time.now.utc.to_i })
+      end
+
+      # Announces an error during a sync and updates the last sync
+      # state and time.
+      def on_error error
+        super(error)
+        Wukong::Deploy.vayacondios_client.announce(vayacondios_topic, {
+          success: false,
+          step:    'prepare',
+          error:   error.class,
+          message: error.message,
+          counts:   counts,
+          files:    files,
+        })
+        Wukong::Deploy.vayacondios_client.set(vayacondios_topic, "prepare.last", { state: 0, time: Time.now.utc.to_i })
+      end
+      
+      # Returns the Vayacondios topic for this PrepareSyncer.
+      #
+      # @return [String] the Vayacondios topic
+      def vayacondios_topic
+        "listeners.#{name}"
+      end
+
+      # Returns a representation of this PrepareSyncer suitable for a
+      # Vayacondios stash.
+      #
+      # @return [Hash]
+      def to_vayacondios
+        {
+          split:     settings[:split],
+          lines:     settings[:lines],
+          bytes:     settings[:bytes],
+          ordered:   settings[:ordered],
+          metadata:  settings[:metadata],
+        }
+      end
+
+      module HandlerOverride
+
+        # The topic for this Handler.
+        #
+        # Delegates to PrepareSyncer#vayacondios_topic.
+        #
+        # @return [String]
+        def vayacondios_topic
+          syncer.vayacondios_topic
+        end
+
+        # Announce the file was processed.
+        #
+        # @param [Pathname] original
+        def after_process original
+          super(original)
+          Wukong::Deploy.vayacondios_client.announce(vayacondios_topic, {
+            success: true,
+            step:    'prepare',
+            path:    fragment_for(original),
+            size:    File.size(original),
+          })
+        end
+
+        # Announce an error in processing a file.
+        #
+        # @param [Pathname] original
+        # @param [Error] error
+        def on_error original, error
+          super(original, error)
+          Wukong::Deploy.vayacondios_client.announce(vayacondios_topic, {
+            success: false,
+            step:    'prepare',
+            path:    fragment_for(original),
+            error:   error.class,
+            message: error.message
+          })
+        end
+        
+      end
+    end
+  end
+end

data/lib/wukong-deploy/extensions/s3_syncer.rb

@@ -0,0 +1,59 @@
+module Wukong
+  module Deploy
+
+    # Attaches to the hooks provided by the Wukong::Load::Syncer class
+    # to write data to Vayacondios.
+    module S3SyncerOverride
+
+      # Saves the syncer as a stash in Vayacondios.
+      def before_sync
+        super()
+        Wukong::Deploy.vayacondios_client.set(vayacondios_topic, 's3', self)
+      end
+
+      # Announces a successful sync and updates the last sync state
+      # and time.
+      def after_sync
+        super()
+        Wukong::Deploy.vayacondios_client.announce(vayacondios_topic, {
+          success: true,
+          step:    's3',
+          bytes: (bytes || 0),
+        }.tap { |e| e[:duration] = duration if duration })
+        Wukong::Deploy.vayacondios_client.set(vayacondios_topic, "s3.last", { state: 1, time: Time.now.utc.to_i })
+      end
+
+      # Announces an error during a sync and updates the last sync
+      # state and time.
+      def on_error error
+        super(error)
+        Wukong::Deploy.vayacondios_client.announce(vayacondios_topic, {
+          success: false,
+          step:    's3',
+          error:   error.class,
+          message: error.message,
+          bytes:   bytes,
+        })
+        Wukong::Deploy.vayacondios_client.set(vayacondios_topic, "s3.last", { state: 0, time: Time.now.utc.to_i })
+      end
+      
+      # Returns the Vayacondios topic for this S3Syncer.
+      #
+      # @return [String] the Vayacondios topic
+      def vayacondios_topic
+        "listeners.#{name}"
+      end
+
+      # Returns a representation of this S3Syncer suitable for a
+      # Vayacondios stash.
+      #
+      # @return [Hash]
+      def to_vayacondios
+        {
+          bucket:    s3_uri,
+          region:    settings[:region],
+        }
+      end
+    end
+  end
+end

data/lib/wukong-deploy/extensions/storm_invocation_override.rb

@@ -0,0 +1,13 @@
+module Wukong
+  module Deploy
+    module StormInvocationOverride
+      def params_to_pass
+        Deploy.pre_deploy_settings
+      end
+    end
+  end
+end
+
+
+    
+    

data/lib/wukong-deploy/extensions/uses_file_state.rb

@@ -0,0 +1,17 @@
+module Wukong
+  module Deploy
+
+    # Makes `UsesFileState#file_state_dir` default to the
+    # `Wukong::Deploy.tmp_dir`.
+    module UsesFileStateOverride
+
+      # Overrides the default value of #file_state_dir with
+      # Wukong::Deploy.tmp_dir.
+      #
+      # @return [String]
+      def file_state_dir
+        Wukong::Deploy.tmp_dir.to_s
+      end
+    end
+  end
+end

data/lib/wukong-deploy/extensions/uses_lockfile.rb

@@ -0,0 +1,17 @@
+module Wukong
+  module Deploy
+
+    # Makes `UsesLockfile#lockfile_dir` default to the
+    # `Wukong::Deploy.tmp_dir`.
+    module UsesLockfileOverride
+
+      # Overrides the default value of #lockfile_dir with
+      # Wukong::Deploy.tmp_dir.
+      #
+      # @return [String]
+      def lockfile_dir
+        Wukong::Deploy.tmp_dir.to_s
+      end
+    end
+  end
+end

data/lib/wukong-deploy/repo.rb

@@ -68,6 +68,8 @@ module Wukong
          config/environments/test.yml
          config/environments/production.yml
 
+         config/initializers/event_machine.rb
+
          Gemfile
          Rakefile
          README.md

data/lib/wukong-deploy/storm_extensions.rb

@@ -0,0 +1,2 @@
+require_relative('extensions/storm_invocation_override')
+Wukong::Storm::StormRunner.class_eval { include Wukong::Deploy::StormInvocationOverride }

data/lib/wukong-deploy/sync_extensions.rb

@@ -0,0 +1,27 @@
+require_relative('extensions/uses_lockfile')
+require_relative('extensions/uses_file_state')
+require_relative('extensions/ftp_syncer')
+require_relative('extensions/prepare_syncer')
+require_relative('extensions/s3_syncer')
+
+Wukong::Load::SyncRunner.class_eval do
+  include Wukong::Deploy::UsesLockfileOverride
+end
+if defined?(Wukong::Load::SyncAllRunner)
+  Wukong::Load::SyncAllRunner.class_eval do
+    include Wukong::Deploy::UsesLockfileOverride
+  end
+end
+Wukong::Load::FTPSyncer.class_eval do
+  include Wukong::Deploy::FTPSyncerOverride
+end
+Wukong::Load::PrepareSyncer.class_eval do
+  include Wukong::Deploy::UsesFileStateOverride
+  include Wukong::Deploy::PrepareSyncerOverride
+end
+Wukong::Load::PrepareSyncer::Handler.class_eval do
+  include Wukong::Deploy::PrepareSyncerOverride::HandlerOverride
+end
+Wukong::Load::S3Syncer.class_eval do
+  include Wukong::Deploy::S3SyncerOverride
+end

data/lib/wukong-deploy/tasks.rb

@@ -1,7 +1,12 @@
 require 'rake'
 
+require 'wukong/rake_helper'
+
 require 'yard'
-YARD::Rake::YardocTask.new(:doc)
+YARD::Rake::YardocTask.new(:doc) do |t|
+  t.files   = %w[app/**/*.rb Rakefile Gemfile]
+  t.options = ['--load', Wukong.doc_helpers_path]
+end
 
 require 'rspec/core/rake_task'
 RSpec::Core::RakeTask.new(:spec)

data/lib/wukong-deploy/version.rb

@@ -1,5 +1,5 @@
 module Wukong
   module Deploy
-    VERSION = '0.1.1'
+    VERSION = '0.2.0'
   end
 end

data/templates/Gemfile.erb

@@ -1,5 +1,5 @@
-# Use the public [RubyGems server](http://rubygems.org/)
-source                 :rubygems
+# Use the public [RubyGems server](https://rubygems.org/)
+source                 'https://rubygems.org'
 
 # = Application
 # 
@@ -12,7 +12,7 @@ source                 :rubygems
 #
 # here and then remember to run
 #
-#   $ bundle install --standalone
+#   $ bundle install
 #
 # Here's a link to the [Bundler documentation](http://gembundler.com/)
 # for your convenience.
@@ -46,6 +46,7 @@ gem 'wonderdog',              '0.1.0'
 # Provides
 #
 #   - wu-storm
+#   - wu-bolt
 gem 'wukong-storm',           '0.1.0'
 
 # Lets you load JSON blobs into data stores.
@@ -56,13 +57,50 @@ gem 'wukong-storm',           '0.1.0'
 #   - wu-source
 gem 'wukong-load',            '0.1.0'
 
-# Framework for gluing code together.
+# If you want to use wu-load with certain databases you may have to
+# enable the various gems required for the database(s) you're
+# interested in here.
+group :load do
+
+  #
+  # Already required by one or more Wukong plugins:
+  #
+  #   - Kafka
+  #
+  # Not required because uses Ruby standard library to connect:
+  #
+  #   - ElasticSearch
+  #
+  
+  # Enable MongoDB
+  # 
+  # gem 'mongo'
+  # gem 'bson_ext'
+
+  # Enable MySQL
+  # 
+  # gem 'mysql2'
+end
+
+# Framework for gluing Wukong plugins together into a deploy pack.
 #
 # Provides
 #
 #   - wu-deploy
 gem 'wukong-deploy',          '0.1.0'
 
+# Provides metadata about a deploy pack.
+#
+# Provides
+#
+#  - wu-meta
+#
+gem 'wukong-meta',            '0.1.0'
+
+# Integrates Wukong processors with Vayacondios.
+#
+gem 'wukong-vayacondios',     '0.1.0'
+
 group :development do
   gem 'yard'
   gem 'redcarpet'

data/templates/README.md.erb

@@ -94,55 +94,84 @@ want to run `rbenv exec bundle install`.
 
 ### Configuration
 
-Your deploy pack doesn't need any configuration out of the box.  As
-you begin to extend it you may add functionality which benefits from
-the ability to be configured.
+#### Configuring the Environment
 
-Put any configuration you want shared across all environments into the
-file `config/settings.yml`.  Override this with environment-specific
-configuration in the appropriate file within `config/environments`.
+Before any of the `wu` programs can run, the Ruby process must first
+boot up, require Wukong and all necessary dependencies (such as
+'event-machine') and plugins (such as the deploy pack plugin
+`wukong-deploy`), and then hand over control to the `wu` program.
 
-As an example, you may write a processor like this:
+The following Ruby files are loaded in order.  Each file is
+responsible for configuring some part of this runtime environment:
+
+1. `config/environment` -- requires the rest of the files and adds any additional environmental code
+2. `config/application` -- defines the load order of external libraries, Wukong plugins, and application code
+3. `config/boot` -- defines how and where the Ruby process will look for code dependencies (through Bundler)
+4. `config/initializers/*.rb` -- non-Wukong configuration for external libraries or application code can live here
+
+#### Configuring the Application
+
+The application a given deploy pack is running can be configured at
+several different layers.
+
+The simplest layer is settings passed to `wu` programs on the
+command-line.  These settings have the highest precedence and will
+always be read.
+
+When booting any of the `wu` tools the deploy pack will also read and
+merge settings from the following configuration files, in order of
+**increasing** precedence:
+
+1. `config/settings.yml`
+2. `config/settings/*.yml` if present, without any guarantee as to order
+3. `config/environments/[environment].yml`
+4. `config/environments/[environment]/*.yml` if present, without any guarantee as to order
+5. `config/deploy.yml` if present (this file should be ignored by version control)
+6. `config/environments/deploy-[environment].yml` if present (this file should be ignored by version control)
+
+Finally, if interaction with Vayacondios is turned on, settings will
+also be read from a Vayacondios stash (see the <a
+href="#vayacondios>Vayacondios section</a> below).
+
+Completely merged and resolved configuration settings are accessible
+globally (once the Wukong framework has been booted) via the
+`Wukong::Deploy.settings` object.  Any piece of code in a model,
+processor, dataflow, or elsewhere can read and write to this object.
+
+Processors will *automatically* read settings for their fields from a
+subhash within this global settings object.  Given a processor like
 
 ```ruby
-Wukong.procesor(:configurable_decorator) do
-  field :suffix, String, :default => '.'
-  def process record
-    yield [record, suffix].join
+Wukong.processor(:tokenizer) do
+  field :min_length, Integer, default: 2
+  def process line
+    ...
   end
 end
 ```
 
-This processor's `suffix` property can be set on the command-line:
-
-```
-$ cat input
-1
-2
-3
-$ cat input | wu-local configurable_decorator
-1.
-2.
-3.
-$ cat input | wu-local configurable_decorator --suffix=','
-1,
-2,
-3,
-
-You can also set the same property in a configuration file, scoped by
-the name of the processor:
+you can set override the value of its `min_length` field by putting
+the following section into any one of the configuration files above:
 
 ```yaml
-# in config/settings.yml
 ---
+# in config/settings.yml, for example
 
-configurable_decorator:
-  suffix: ,
+tokenizer:
+  min_length: 5
 ```
 
-which lets you the `--suffix` flag on the command-line while still
-overriding the default setting.  You can also put such settings in
-environment specific files within `config/environments`.
+which would now make the command
+
+```
+$ cat corpus.txt | wu local tokenizer
+```
+
+have the same effect as
+
+```
+$ cat corpus.txt | wu local tokenizer --min_length=5
+```
 
 ## File Structure
 
@@ -206,3 +235,200 @@ the underlying documentation for Wukong and its plugins, specifically:
 
 * on [Wukong](http://github.com/infochimps-labs/wukong/tree/3.0.0) so you understand the basic idea of a processor and how to glue processors together
 * on [Wukong-Hadoop](http://github.com/infochimps-labs/wukong-hadoop) so you understand how to move between local and Hadoop modes for batch analytics
+
+
+<a target="#vayacondios">
+## Interacting with Vayacondios
+
+[Vayacondios](http://github.com/infochimps-labs/vayacondios) is a
+program which makes it easy to for clients to announce events or read
+and write settings to and from a central server.
+
+The basic objects of Vayacondios are **stash** and the **event**:
+
+* a **stash** is an "object", a "configuration", or "setting" designed to be shared among many services
+* an **event** is a "fact", "measurement", or "metric" announced by an arbitrary service, possibly related to some stash
+
+Stashes and events are organized in two levels.
+
+The top-level is the **organization**.  Data from multiple
+organizations is stored together but accessed separately by a running
+Vayacondios server.  An organization could be the name of a user,
+workgroup, application, or service using Vayacondios.
+
+The next level is the **topic**.  Each topic within Vayacondios has a
+single stash and can have multiple events.  An "object" like a server,
+a database, an application, a service, or a user maps to the concept
+of "topic".
+
+Every `wu` tool running within a deploy pack takes an additional
+option `--vcd` which turns on or off interactions with Vayacondios.
+This option can be specified at runtime on the command-line as well as
+via a configuration file.  When not running "in Vayacondios mode"
+(with `--vcd` was not passed), interactions with Vayacondios will be
+logged instead of transmitted and received.
+
+### Configuring Vayacondios access
+
+If you don't intend to interact with a Vayacondios server, you can
+just set `vcd` to `false` for your whole environment and skip this
+section (as is done, for example, in the `test` environment by
+default).
+
+If you intend to interact with Vayacondios then you need to also
+specify the `vcd_host` and `vcd_port` options which otherwise default
+to the usual Vayacondios server port running on localhost.
+
+```yaml
+---
+# in config/environments/production.yml
+vcd_host: 10.123.123.123
+vcd_port: 9000
+```
+
+Vayacondios also requires that all events and stashes are stored under
+a given organization name.  The Vayacondios organization, which will
+likely be shared across all environments of your application, is
+usually set at the top-level:
+
+```yaml
+---
+# in config/settings.yml
+organization: my_company
+```
+
+### Handle out of band event data with Events
+
+Despite being designed to be powerful and scalable, Vayacondios is not
+the appropriate store for high-volume, high-throughput,
+mission-critical data which must be persisited over the long-term.
+Instead it should be used for "out of band" data, which is typically
+much smaller in volume and throughput than the main body of a
+dataflow.  Examples of such out of band events include:
+
+* signalling some intermittend or runtime error
+* warning that some event was bad or suspicious
+* logging an error
+* registering some periodic metric
+* signaling a change in state
+
+Announcements can be made from anywhere within the Wukong framework by
+accessing the `Wukong::Deploy.vayacondios_client` object but the most
+common approach is to announce events within a processor or within a
+dataflow.
+
+#### Announcing from a processor
+
+The `Wukong::Processor#announce` method can be used to directly send
+an event to Vayacondios on a given topic.
+
+```ruby
+Wukong.processor(:parser) do
+  def process line
+    yield parse!(line)
+  rescue ParseError => e
+    announce "parser.errors", line: line
+  end
+end
+```
+
+It's important when setting up an announcement like this that you
+consider how often this piece of code will actually send events to
+Vayacondios.  If a `ParseError` is triggered once in every 10,000
+lines, this may be perfectly fine to be running in production.  If 1
+in 10 lines causes a similar error, this may not be the right
+approach.
+
+#### Announcing from a dataflow
+
+The `announce` processor can be used to send all announce all incoming
+events to Vayacondios.  Here's an example flow which makes use of it:
+
+```ruby
+Wukong.dataflow(:parse_source) do
+  parser |
+  [
+    select(&:valid?) | ... | to_json,
+	select(&:invalid?) | announce(topic: "invalid_records")
+  ]
+end
+```
+
+Just as in the above example with a processor, it's important that the
+flow through the announce processor is not incredibly high-volume.
+
+The `announce` processor is terminal; it yields no output records.
+
+### Allow dynamic configuration with Stashes
+
+The deploy pack inside a backend system like Hadoop or Storm can fetch
+stashes from Vayacondios during runtime.  Other systems external to
+the deploy pack can simultaneously be writing data into these same
+stashes in Vayacondios, allowing for a lightweight, two-way
+communication stream between the deploy pack and arbitrary external
+resources, mediated by a key-value store (the Vayacondios stash).
+
+Stashes can be read and written from anywhere within the Wukong
+framework by accessing the `Wukong::Deploy.vayacondios_client` object
+but there are two special places where encapsulated, remote settings
+are very useful.
+
+#### Dynamic settings for the deploy pack itself
+
+Each deploy pack, as an application, can fetch a stash of settings
+from Vayacondios and use this as bootup time in the same way it uses a
+configuration file ond disk.  All that is required is a Vayacondios
+stash topic name.  This is furnished by providing to the deploy pack
+an `application` name in a configuration file, usually the top-level
+one:
+
+```yaml
+---
+# in config/settings.yml, for example
+
+application: my_app
+```
+
+When any `wu` tool is launched within the deploy pack with the `--vcd`
+option (possibly set an an environment-wide level via a configuration
+file) then remote settings from Vayacondios for the `application` will
+be pulled at boot-time and merged into the local settings from
+configuration files and the command-line.
+
+#### Dynamic settings for processors
+
+The processor `tokenizer` in the deploy pack with application name
+`my_app` defaults to using the stash with topic
+`processors.my_app-tokenizer` in Vayacondios to store its settings
+(this can be changed by overriding the `Wukong::Processor#vcd_topic`
+method).
+
+These settings, if they exist, can be retrieved and merged into the
+processor's current fields at anytime using the
+`Wukong::Processor#update_settings`.  A common use case is to want to
+update a processor's fields every 30 seconds, or similar.  This is
+most easily accomplished via the
+`Wukong::Processor#update_settings_every` method.  Here's an example
+
+```ruby
+Wukong.processor(:tagger) do
+  field :tags, Array, doc: "List of tags to check", default: []
+
+  def setup
+    update_settings_every(30)
+  end
+  
+  def process record
+    tags.each do |tag|
+	  ...
+	end
+  end
+end
+```
+
+The `tags` field of this processor will be updated every 30 seconds
+with the latest values from Vayacondios.
+
+The `Wukong::Processor#save_settings` and
+`Wukong::Processor#save_settings_every` and methods can be used to
+save settings from a processor **to** Vayacondios.

data/templates/config/boot.rb.erb

@@ -1,17 +1,6 @@
 # = config/boot.rb
 #
 # Loads the Ruby language framework and activates Bundler.
-#
-# We can use a system-wide bundle installation _or_ a standalone
-# bundle installation.  If a standalone installation exists (at
-# /bundle/bundler/setup.rb), we will detect and use it.  Otherwise, we
-# fall back on the system-wide bundle installation.
-#
-# Make sure you install the bundle locally with the following command:
-#
-# @example Install a standalone bundle of gems
-#
-#   $ bundle install --standalone
 
 local_setup_path = File.expand_path("../../bundle/bundler/setup.rb", __FILE__)
 if File.exist?(local_setup_path)

data/templates/config/initializers/event_machine.rb.erb

@@ -0,0 +1,5 @@
+# EventMachine hard-codes a very inconvenient maximum length for a
+# line -- 16K.  This is too small for many use cases and is enlarged
+# here to 16M.
+EventMachine::Protocols::LineAndTextProtocol.send(:remove_const, :MaxLineLength)
+EventMachine::Protocols::LineAndTextProtocol.send(:const_set,    :MaxLineLength, 16*1024*1024)

data/templates/gitignore

@@ -35,5 +35,10 @@ a.out
 /doc/*
 /pkg/*
 
+# bundler -- do NOT add Gemfile.lock to this list
 .bundle
 bundle/*
+
+# configuration files that are dropped in outside of git
+config/development.yml
+config/deploy.yml

data/wukong-deploy.gemspec

@@ -31,11 +31,11 @@ Gem::Specification.new do |gem|
   EOF
 
   gem.files         = `git ls-files`.split("\n")
-  gem.executables   = ['wu-deploy','wu']
+  gem.executables   = ['wu-deploy']
   gem.test_files    = gem.files.grep(/^spec/)
   gem.require_paths = ['lib']
 
-  gem.add_dependency('wukong',        '3.0.1')
+  gem.add_dependency('wukong',        '4.0.0')
   gem.add_dependency('rake',          '~> 0.9')
   gem.add_dependency('erubis')
   gem.add_dependency('diffy')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wukong-deploy
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-07 00:00:00.000000000 Z
+date: 2014-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: wukong
@@ -20,7 +20,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.0.1
+        version: 4.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -28,7 +28,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.0.1
+        version: 4.0.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -91,7 +91,6 @@ description: ! "  The Infochimps Platform is an end-to-end, managed solution for
 email: coders@infochimps.com
 executables:
 - wu-deploy
-- wu
 extensions: []
 extra_rdoc_files: []
 files:
@@ -102,17 +101,25 @@ files:
 - LICENSE.md
 - README.md
 - Rakefile
-- bin/wu
 - bin/wu-deploy
 - examples/.gitkeep
 - lib/wukong-deploy.rb
 - lib/wukong-deploy/console.rb
 - lib/wukong-deploy/deploy_pack.rb
 - lib/wukong-deploy/deploy_runner.rb
-- lib/wukong-deploy/extensions.rb
+- lib/wukong-deploy/dump_extensions.rb
+- lib/wukong-deploy/extensions/ftp_syncer.rb
 - lib/wukong-deploy/extensions/hadoop_invocation_override.rb
+- lib/wukong-deploy/extensions/prepare_syncer.rb
+- lib/wukong-deploy/extensions/s3_syncer.rb
+- lib/wukong-deploy/extensions/storm_invocation_override.rb
+- lib/wukong-deploy/extensions/uses_file_state.rb
+- lib/wukong-deploy/extensions/uses_lockfile.rb
+- lib/wukong-deploy/hadoop_extensions.rb
 - lib/wukong-deploy/rake_runner.rb
 - lib/wukong-deploy/repo.rb
+- lib/wukong-deploy/storm_extensions.rb
+- lib/wukong-deploy/sync_extensions.rb
 - lib/wukong-deploy/tasks.rb
 - lib/wukong-deploy/templater.rb
 - lib/wukong-deploy/templater/conflict_resolution.rb
@@ -131,6 +138,7 @@ files:
 - templates/config/environments/development.yml.erb
 - templates/config/environments/production.yml.erb
 - templates/config/environments/test.yml.erb
+- templates/config/initializers/event_machine.rb.erb
 - templates/config/settings.yml.erb
 - templates/gitignore
 - templates/spec/spec_helper.rb.erb
@@ -150,7 +158,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 687970152159308186
+      hash: -3128770076158943322
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -159,10 +167,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 687970152159308186
+      hash: -3128770076158943322
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Defines the deploy pack framework used by the Infochimps Platform

data/bin/wu

@@ -1,35 +0,0 @@
-#!/usr/bin/env ruby
-require 'shellwords'
-now=Time.now.strftime("%Y-%m-%d %H:%M:%S")
-if ARGV.empty?
-    $stderr.puts "ERROR #{now} [wu                  ] -- Must provide a Wukong command to run. Try the --help option."
-else
-    if ARGV.size == 1 && ARGV.first == '--help'
-	$stderr.puts <<EOF
-usage: wu COMMAND [OPTIONS] [ARG] ...
-
-wu is a wrapper for easy use of Wukong's command-line tools.  It takes
-your arguments, constructs the name of the proper wu-tool to call, and
-prepends a call to bundle exec.
-
-  $ wu local ...
-
-is equivalent to
-
-  $ bundle exec wu-local ...
-
-You can run any of the wu-tools like
-	
-  wu-local
-  wu-hadoop
-  wu-deploy
-
-EOF
-    else
-      if ARGV.first =~ /^-/
-        $stderr.puts "ERROR ${now} [wu                  ] -- First argument must be the name of a wu tool to run, got <${1}>"
-      else
-        Kernel.exec "bundle exec wu-#{Shellwords.join(ARGV)}"
-      end
-    end
-end