continuent-tools-core 0.9.0 → 0.10.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e7f7d7c19ce143f710bec3d7b7c0548c76e2cc4b
-  data.tar.gz: c8fc19d558b5e084195527b21737623a8d56dc3d
+  metadata.gz: 98ee92f1a8d4f2e609211c326b0fe054e9a048ad
+  data.tar.gz: 1a99ad83f75681fca485e740677e876d56fc519f
 SHA512:
-  metadata.gz: 28857957400e927922005705c025814245cf72d3a8118ba0afedd4ab136c150701baba607aea3b3e245876dd82ffc68b6823f0be43b359d685d80e86c3ed7e71
-  data.tar.gz: daa48a69f3a8c21c627add56e9ecd9a187386c45031bef2d8582b9e855fc6b0b0620f94ac678df4f2dbb467f9c0bd4f01c2282fda386559f4e6d740d1e8830bd
+  metadata.gz: e97a7d38b5089aa673929764ccac1fa0331921359143e241148d3e693288163c01051a5f13150e80a81996a592ffc167b662fa46d48b870fbe13c4488374295e
+  data.tar.gz: 9da41564c3b4534072dac80f4680fa207b95d6841e7921c5006d034103282d2c859c641b612ae995b7ccd33d0465aec668879dc73a1f065ab20a9834aad0e6ff

data/README.md

@@ -1,9 +1,30 @@
 continuent-tools-core
 =====================
 
-The continuent-tools-core is a package of libraries that are found in Tungsten Replicator. When the gem is built, these files are automatically exported from [https://code.google.com/p/tungsten-replicator/](https://code.google.com/p/tungsten-replicator/).
+The continuent-tools-core is a package of libraries and scripts to use with most Continuent Tungsten deployments. It is the basis for all other RubyGems distributed by Continuent.
 
-    gem install continuent-tools-core
+The code is focused around the 'TungstenScript' Ruby class that is found in Tungsten Replicator. When the gem is built, these files are automatically exported from [https://code.google.com/p/tungsten-replicator/](https://code.google.com/p/tungsten-replicator/).
+
+Installation may be done using the `gem` utility. This will ensure that all prerequisites are installed.
+
+    $> gem install continuent-tools-core
+    
+Using the tungsten\_create\_load script
+===
+
+The tungsten\_create\_load script may be used to apply load directly to a MySQL server or through a Tungsten Connector. By default, the script will read your configuration and use the Tungsten Connector if it is available. This may be disabled by adding '--use-connector=false'.
+
+Automated Continuent Tungsten management scripts
+===
+
+The automated Continuent Tungsten management scripts are designed to be used as a way to manage Continuent Tungsten host configurations. They use provided or discovered directory information to determine the configuration of the local host and install or update the necessary components.
+
+These tools can be called using CRON, Puppet, Chef or other devops tools with little customization to the specific platform.
+
+See [MANAGE\_CONFIGURATION.md](MANAGE\_CONFIGURATION.md) for more details.
+
+Using the TungstenScript class
+===
 
 Script Structure
 ---

data/bin/tungsten_analyze_thl_index

@@ -50,15 +50,28 @@ class TungstenParseTHLIndex
   end
   
   def find_thl_record
-    each_thl_index_line() {
-      |line|
-      match = parse_thl_index_line(line)
-      if match != nil
-        if match[:start] <= opt(:seqno) && match[:end] >= opt(:seqno)
-          TU.output(match[:file])
+    filename = nil
+    
+    begin
+      each_thl_index_line() {
+        |line|
+        match = parse_thl_index_line(line)
+        if match != nil
+          if match[:start] <= opt(:seqno) && match[:end] >= opt(:seqno)
+            filename = match[:file]
+            raise IgnoreError.new()
+          end
         end
-      end
-    }
+      }
+    rescue IgnoreError
+      # Do Nothing
+    end
+    
+    if filename != nil
+      TU.output(filename)
+    else
+      TU.error("Unable to find THL event ##{opt(:seqno)}")
+    end
   end
   
   def parse_thl_index_line(line)
@@ -76,14 +89,14 @@ class TungstenParseTHLIndex
   end
   
   def each_thl_index_line(&block)
-    TU.cmd_stdout("cat /Users/jmace/Downloads/thl.txt") {
+    TU.cmd_stdout("#{TI.thl(opt(:service))} index") {
       |line|
       block.call(line.strip())
     }
   end
   
   def get_thl_index
-    return TU.cmd_result("cat /Users/jmace/Downloads/thl.txt").split("\n")
+    return TU.cmd_result("#{TI.thl(opt(:service))} index").split("\n")
   end
   
   def configure

data/bin/tungsten_create_load

@@ -54,7 +54,16 @@ class ContinuentCreateLoad
     # generation thread.
     load_threads.peach{
       |load_thread|
-      TU.cmd(get_mysql_command() + " -h#{load_thread[:host]}", true, method(:create_schema_load), nil, method(:forward_mysql_errors))
+      while (ContinuentCreateLoad.interrupted?() == false)
+        begin
+          TU.cmd(get_mysql_command() + " -h#{load_thread[:host]}", true, method(:create_schema_load), nil, method(:forward_mysql_errors))
+        rescue => e
+          TU.debug(e)
+        end
+        if ContinuentCreateLoad.interrupted?() == false
+          TU.notice("Reconnecting to the closed MySQL connection.")
+        end
+      end
     }
     
     puts("\n")
@@ -65,7 +74,12 @@ class ContinuentCreateLoad
     [
       "DROP SCHEMA IF EXISTS tungsten_create_load;",
       "CREATE SCHEMA tungsten_create_load;",
-      "CREATE TABLE tungsten_create_load.values (id int NOT NULL auto_increment primary key, val int NOT NULL, origin varchar(32) NULL);"
+      "CREATE TABLE tungsten_create_load.#{opt(:table_name)} (
+        id int NOT NULL auto_increment primary key, 
+        val int NOT NULL, 
+        origin varchar(32) NULL,
+        filler varchar(1024) NULL
+      );"
     ].each{|sql|
       stdin.puts(sql)
       putc '.'
@@ -74,13 +88,15 @@ class ContinuentCreateLoad
   end
   
   def create_schema_load(stdin)
-    sql = "INSERT INTO tungsten_create_load.values (val, origin) VALUES (5, @@hostname);"
+    value = ""; 1024.times{value << ((rand(2)==1?65:97) + rand(25)).chr}
+    sql = "INSERT INTO tungsten_create_load.#{opt(:table_name)} (val, origin, filler) VALUES (5, @@hostname, '#{value}');"*[opt(:chunk_size),1].max()
     while (ContinuentCreateLoad.interrupted?() == false)
       stdin.puts(sql)
       putc '.'
       $stdout.flush()
       sleep opt(:sleep).to_i()
     end
+    stdin.puts("exit")
   end
   
   def forward_mysql_errors(msg)
@@ -102,6 +118,13 @@ class ContinuentCreateLoad
       :default => 1,
     })
     
+    add_option(:chunk_size, {
+      :on => "--chunk-size String",
+      :parse => method(:parse_integer_option),
+      :help => "How many rows should be entered per iteration",
+      :default => 10,
+    })
+    
     add_option(:sleep, {
       :on => "--sleep String",
       :parse => method(:parse_integer_option),
@@ -114,6 +137,12 @@ class ContinuentCreateLoad
       :parse => method(:parse_boolean_option),
       :help => "Enable/Disable use of the Tungsten Connector for adding load to the system.",
     })
+    
+    add_option(:table_name, {
+      :on => "--table-name String",
+      :help => "The MySQL table name to fill with this script",
+      :default => "values"
+    })
   end
   
   def validate
@@ -123,7 +152,7 @@ class ContinuentCreateLoad
       return TU.is_valid?()
     end
     
-    if opt(:use_connector) == nil
+    if opt(:use_connector) == nil || opt(:use_connector) == true
       if TI.is_connector?()
         opt(:use_connector, true)
         
@@ -139,14 +168,17 @@ class ContinuentCreateLoad
         defaults_file.puts("password=#{TI.setting(TI.setting_key(CONNECTORS, "connector_password"))}")
         defaults_file.flush()
         
-        opt(:mysqlport, TI.setting(TI.setting_key(CONNECTORS, "connector_listen_port")))
         opt(:mysqlhost, TI.hostname())
+        opt(:mysqlport, TI.setting(TI.setting_key(CONNECTORS, "connector_listen_port")))
       elsif TI.is_replicator?()
         opt(:use_connector, false)
         
         if TI.setting(TI.setting_key(REPL_SERVICES, @options[:service], "repl_datasource_type")) != "mysql"
           TU.error("Unable to create load on this system because it is not configured for MySQL")
         end
+        
+        opt(:mysqlhost, TI.setting(TI.setting_key(REPL_SERVICES, opt(:service), "repl_direct_datasource_host")))
+        opt(:mysqlport, TI.setting(TI.setting_key(REPL_SERVICES, opt(:service), "repl_direct_datasource_port")))
       else
         TU.error("Unable to create load on this system because it is not configured as a Tungsten Connector or Tungsten Replicator")
       end
@@ -157,10 +189,9 @@ class ContinuentCreateLoad
         |h|
         (h == TI.hostname())
       }
-      hosts << TI.hostname()
       opt(:hosts, hosts)
     else
-      opt(:hosts, [TI.hostname()])
+      opt(:hosts, [opt(:mysqlhost)])
     end
   end
   

data/bin/tungsten_directory

@@ -0,0 +1,379 @@
+#!/usr/bin/env ruby
+# Copyright (C) 2014 Continuent, Inc.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.  You may obtain
+# a copy of the License at
+# 
+#         http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Initial developer(s): Jeff Mace
+# Contributor(s):
+
+# TODO : Add commands to output host definitions in Chef form
+
+begin
+  require 'rubygems'
+  gem 'continuent-tools-core'
+rescue LoadError
+end
+
+require 'continuent-tools-core'
+
+class TungstenDirectoryProvider
+  def initialize(key)
+    @key = key
+  end
+  
+  def self.inherited(subclass)
+    @subclasses ||= []
+    @subclasses << subclass
+  end
+  
+  def self.subclasses
+    @subclasses
+  end
+  
+  def self.get_provider(key)
+    @subclasses.each{
+      |klass|
+      
+      regex = Regexp.new(klass.get_regex())
+      if key =~ regex
+        return klass.new(key)
+      end
+    }
+    
+    nil
+  end
+end
+
+Dir.glob(File.dirname(__FILE__) + '/../providers/*.rb').each do |file| 
+  begin
+    require file
+  rescue IgnoreError
+  end
+end
+
+class TungstenDirectory
+  include TungstenScript
+  private
+  
+  def main
+    directory_entries = collect_directory_entries()
+    
+    unless TU.is_valid?
+      return
+    end
+    
+    case command()
+    when "list"
+      TU.output(JSON.pretty_generate(directory_entries))
+    when "hosts"
+      if directory_entries.has_key?(opt(:hostname))
+        location = directory_entries[opt(:hostname)]["location"]
+      else
+        location = nil
+      end
+      hostsmap = generate_hosts_map(directory_entries, location)
+      hostsmap.keys().sort().each{
+        |h|
+        TU.output("#{hostsmap[h]}\t#{h}")
+      }
+    when "hosts_puppet_manifest"
+      if directory_entries.has_key?(opt(:hostname))
+        location = directory_entries[opt(:hostname)]["location"]
+      else
+        location = nil
+      end
+      hostsmap = generate_hosts_map(directory_entries, location)
+      hostsmap.keys().sort().each{
+        |h|
+        TU.output("host { '#{h}' : ip => '#{hostsmap[h]}', comment => 'Created by #{script_name()}'}")
+      }
+    end
+  end
+  
+  def collect_directory_entries
+    found_files = false
+    directory_entries = {}
+    
+    Dir.glob(opt(:config)).each{
+      |f|
+      found_files = true
+      
+      # Parse the configuration file and remove any sections
+      # that may hold command line options
+      contents = TU.parse_ini_file(f)
+      contents.delete("tungsten_directory")
+      contents.delete("directory")
+      
+      # Iterate through each section and merge directory entries
+      contents.each{
+        |k,v|
+        
+        matches = k.match("autodetect.(.*)")
+        if matches != nil and matches.size() > 0
+          # Search for a matching autodetect provider
+          provider = TungstenDirectoryProvider.get_provider(matches[1])
+          unless provider == nil
+            directory_entries.merge!(provider.get_entries(v))
+          else
+            TU.error("Unable to autodetect #{matches[1]} entries")
+          end
+        else
+          entry = parse_directory_entry(v)
+          hostname = k
+          if entry.has_key?("hostname")
+            hostname = entry["hostname"]
+          else
+            entry["hostname"] = hostname
+          end
+          
+          # Build a manual entry and add it to the list
+          directory_entries[k] = entry
+        end
+      }
+    }
+    
+    if found_files == false
+      raise "Unable to find any files at #{opt(:config)}"
+    end
+    
+    # Modify each directory entry and prepare it for output
+    directory_entries.each{
+      |id,entry|
+      unless entry["tags"].is_a?(Hash)
+        next
+      end
+      
+      unless entry["hostname"] != ""
+        TU.error("Entry '#{entry["id"]}' does not include a hostname")
+      end
+      
+      unless entry["location"] != ""
+        TU.error("Entry '#{entry["id"]}' does not include a location")
+      end
+      
+      entry["id"] = id
+      entry["tags"].each{
+        |k,v|
+        v = v.split(",")
+        if v.size() > 1
+          entry["tags"][k] = v
+        end
+      }
+    }
+    
+    if has_filters?()
+      directory_entries = apply_filters(directory_entries, opt(:filters))
+    end
+    
+    return directory_entries
+  end
+  
+  def parse_directory_entry(e)
+    entry = e.dup()
+    entry["tags"] = {}
+    entry["provider"] = "ini"
+    
+    e.keys().each{
+      |k|
+      parts = k.split(".")
+      if parts.size() == 1
+        next
+      end
+      
+      type = parts.shift()
+      remainder = parts.join(".")
+      
+      case type
+      when "tags"
+        entry["tags"][remainder] = entry[k]
+        entry.delete(k)
+      end
+    }
+    
+    return entry
+  end
+  
+  # For each entry include the private address if the host location
+  # matches the given location. Use the public address if there is no private
+  # address or the locations do not match.
+  def generate_hosts_map(directory_entries, location = nil)
+    hosts_map = {}
+    
+    directory_entries.each{
+      |hostname,entry|
+      if location != nil && entry["location"] == location
+        if entry.has_key?("private-address")
+          hosts_map[hostname] = entry["private-address"]
+        elsif entry.has_key?("public-address")
+          hosts_map[hostname] = entry["public-address"]
+        else
+          TU.error("Unable to find a private or public address for #{hostname}")
+        end
+      else
+        if entry.has_key?("public-address")
+          hosts_map[hostname] = entry["public-address"]
+        else
+          TU.error("Unable to find a public address for #{hostname}")
+        end
+      end
+    }
+    
+    return hosts_map
+  end
+  
+  def has_filters?
+    filters = opt(:filters)
+    
+    if filters == nil
+      return false
+    end
+    
+    if filters.is_a?(Array) && filters.count() > 0
+      return true
+    else
+      return false
+    end
+  end
+  
+  # Remove any entry that does not match the provided filters
+  def apply_filters(entries, filters)
+    filters.each{
+      |filter|
+      parts = filter.split("=")
+      
+      # Determine if the filter is joined with a '=' or '!='
+      last_char=parts[0][-1,1]
+      if last_char == "!"
+        test_equals = false
+        key = parts[0][0, parts[0].length()-1]
+        match = parts[1]
+      else
+        test_equals = true
+        key = parts[0]
+        match = parts[1]
+      end
+      
+      entries.each{
+        |e, entry|
+        # Find the key value in entry, or nil if it doesn't exist
+        value = find_entry_setting(entry, key)
+        
+        # Calculate if the returned value matches the filter value
+        if value.is_a?(Array)
+          is_equal = false
+          value.each{
+            |v|
+            if is_value_match?(v, match)
+              is_equal = true
+            end
+          }
+        else
+          is_equal = is_value_match?(value, match)
+        end
+        
+        # Delete the entry if the result of the match check isn't what
+        # the filter is looking for
+        if is_equal != test_equals
+          entries.delete(e)
+        end
+      }
+    }
+    
+    return entries
+  end
+  
+  def is_value_match?(value, match)
+    match = match.sub("%", ".*")
+    matches = value.to_s().match("^#{match}$")
+    if matches == nil
+      return false
+    else
+      return true
+    end
+  end
+  
+  def find_entry_setting(entry, key)
+    attrs = key.split(".")
+    attr_count = attrs.size
+    current_val = entry
+    for i in 0..(attr_count-1)
+      attr_name = attrs[i]
+      return current_val[attr_name] if i == (attr_count-1)
+      return nil if current_val[attr_name].nil?
+      current_val = current_val[attr_name]
+    end
+    
+    return nil
+  end
+  
+  def configure
+    super()
+    
+    require_installed_directory?(false)
+    
+    add_option(:config, {
+      :on => "--config String",
+      :help => "Path to INI file that holds directory information",
+      :default => "/etc/tungsten/directory.ini"
+    })
+    
+    add_option(:hostname, {
+      :on => "--hostname String",
+      :help => "Use this hostname for calculating hosts entries",
+      :default => TU.hostname()
+    })
+    
+    add_option(:filters, {
+      :on => "--filter String"
+    }) {|val|
+      unless @options.has_key?(:filters)
+        @options[:filters] = []
+      end
+
+      parts = val.split("=")
+      if parts.count() != 2
+        TU.error("Unable to parse --filter=#{val}. The filter should be in the form of 'key=value' or 'key!=value'.")
+      end
+
+      @options[:filters] << val
+      nil
+    }
+    
+    add_command(:list, {
+      :help => "Output the directory information as JSON",
+      :default => true
+    })
+    
+    add_command(:hosts, {
+      :help => "Output /etc/hosts entries for the directory hosts"
+    })
+    
+    add_command(:hosts_puppet_manifest, {
+      :help => "Output a Puppet manifest for the directory hosts"
+    })
+  end
+  
+  def validate
+    super()
+    
+    unless TU.is_valid?()
+      return TU.is_valid?()
+    end
+  end
+  
+  def script_name
+    "tungsten_directory"
+  end
+  
+  self.new().run()
+end