poolparty 0.2.84 → 1.2.2

data/bin/cloud-start

@@ -1,32 +1,30 @@
 #!/usr/bin/env ruby
 $:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
 require "poolparty"
-require "poolpartycl"
 
-o = PoolParty::Optioner.new(ARGV) do |opts, optioner|  
-  optioner.cloudnames
-  optioner.daemonizeable
+o = PoolParty::Optioner.new(ARGV) do |opts, optioner|
+  opts.on('-n cloudname', '--name name', 'Start cloud by this name')    { |c| optioner.cloudname c }
 end
 
-o.loaded_clouds.each do |cloud|
-  with_cloud(cloud) do
-    setup_dev
-    vputs header("Starting cloud #{name}")
-    vputs "#{list_of_running_instances.size} running instances (#{minimum_instances} - #{maximum_instances})"
-    # if list_of_running_instances.size <= 1
-      puts "Starting the master instance"    
-      # if testing
-      #   puts("\tNot launching while in testing mode")
-      # else
-        vputs "Launching and configuring the master"
-        @cl = self
-        daemon ? daemonize {@cl.launch_and_configure_master!} : launch_and_configure_master!
-      # end
-    # end
-    when_no_pending_instances do
-      puts open(::File.join(File.dirname(__FILE__), "..", "lib", "poolparty", "config", "postlaunchmessage.txt")).read ^ {:master_ip => master.ip.chomp}
-      clear_base_directory unless testing
+o.loaded_clouds.each do |cld|
+
+  if cld.nodes(:status => "running").size.zero?
+    puts header("Starting cloud #{cld.name} (#{cld.keypair})")
+    puts "#{cld.nodes(:status => "running").size} running instances (#{cld.minimum_instances} - #{cld.maximum_instances})"
+    
+    cld.launch_instance!(cld.dsl_options) do |node|
+      
+      ::PoolParty::Provision::BootStrapper.new(node.ip, :cloud => cld)
+      ::PoolParty::Provision::DrConfigure.new(node.ip,  :cloud => cld)
+      
+      puts <<-EOM
+        Your cloud has started. Your ip is #{node.ip}
+      EOM
     end
-  end
   
+  else
+    puts <<-EOE
+      Your cloud is already running.
+    EOE
+  end
 end

data/bin/cloud-terminate

@@ -5,16 +5,24 @@ require "poolpartycl"
 
 o = PoolParty::Optioner.new(ARGV) do |opts, optioner|
   opts.on('-n name', '--name name', 'Terminate the cloud of this name')    { |c| optioner.cloudname c }
+  opts.on('-y', '--yes', 'Terminate the cloud without prompt')    { optioner.yes true }
 end
 
-o.loaded_clouds.each do |cloud|
-  
-  with_cloud(cloud) do
+o.loaded_clouds.each do |cld|
+  cld.nodes(:status => "running").each do |inst|
+    msg = <<-EOM
+Shutting down #{inst.instance_id}
+Are you sure? (Y/N)
+    EOM
     
-    list_of_running_instances.each do |inst|
+    if o.yes?
       puts "Shutting down #{inst.instance_id}"
-      terminate_instance!(inst.instance_id)
+      cld.terminate_instance!(:instance_id => inst.instance_id)
+    else
+      if are_you_sure?(msg)
+        puts "Shutting down #{inst.instance_id}"
+        cld.terminate_instance!(:instance_id => inst.instance_id)
+      end
     end
-    
-  end    
+  end
 end

data/bin/cloud-verify

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+$:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+require "poolparty"
+
+o = PoolParty::Optioner.new(ARGV) do |opts, optioner|
+  opts.on('-n cloudname', '--name name', 'Start cloud by this name')    { |c| optioner.cloudname c }
+  opts.on('-i num', '--num i', 'Instance number')    { |i| optioner.inst_num i.to_i }
+end
+
+o.loaded_clouds.each do |cld|
+
+  begin
+    if cld.verifiers.size > 0
+      cld.passing?
+      cld.vputs "Cloud passed verification"
+    else
+      raise "Your cloud doesn't have any verifiers!"
+      exit 2
+    end
+  rescue Exception => e
+     cld.vputs <<-EOM
+
+      Verification failed: #{e}"
+        Please check your clouds.rb for any errors
+
+      Exception: #{e.class}: #{e.message}\n\t#{e.backtrace.join("\n\t")}
+
+      EOM
+  end
+
+end
+

data/bin/install-poolparty

@@ -0,0 +1,216 @@
+#!/usr/bin/env ruby
+$:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+require "poolparty"
+require "poolpartycl"
+require "highline/import"
+     
+welcome_msg = <<-EOE
+We are going to take you through the installation process of PoolParty.
+
+First, we'll setup your environment so using PoolParty will be a breeze
+EOE
+
+closing_message = <<-EOE
+You are now set to ride the waves with PoolParty! You'll notice there is a clouds.rb file in your current directory. You can start your new cloud
+by typing:
+
+cloud start
+
+You can start your clouds.rb. More samples are available here: 
+  http://github.com/auser/poolparty-examples/tree/master
+EOE
+
+colored_say "Welcome to PoolParty!", :help
+say welcome_msg
+begin
+  t = colored_ask "Press enter to continue or Ctrl+C to exit"
+rescue Exception => e
+  say <<-EOE
+
+Cancelled PoolParty installation
+
+You can always restart this by typing:
+  install-poolparty
+EOE
+  exit 0
+end
+
+say("\nWhat remoter base would you like to use?")
+choose do |menu|
+  menu.prompt = "> "
+
+  available_bases.each do |base|
+    menu.choice base.to_sym do 
+      
+      colored_say "Great, we'll be using #{base}"
+      
+      case base
+      when :ec2
+        
+        key_help_str = <<-EOE
+Now you need to get your keys from Amazon AWS.
+If you don't already have your keys setup, navigate to http://aws.amazon.com
+and click on Your Account. Click on the Access Identifiers tab and find:
+Your Access Key ID. This is your access key.
+        EOE
+        access_key_str = <<-EOE
+Since you'll be using ec2, we'll have to set some things up before
+we can get going on PoolParty. Don't worry, this information will stay between us.
+
+        EOE
+        say access_key_str
+        ask_with_help :message => "First, what's your access key?",
+                      :help => key_help_str do |t|
+          
+          @access_key = t
+          
+        key_help_str = <<-EOE
+If you don't already have your keys setup, navigate to http://aws.amazon.com
+and click on Your Account. Click on the Access Identifiers tab and find:
+Your Secret Access Key ID. Click on the - show button.
+This is your secret access key.
+        EOE
+          ask_with_help :message => "Awesome. Now what's your secret access key? ",
+                        :help => key_help_str do |t|
+            @secret_access_key = t
+            
+            rescued_ask_str = <<-EOS
+Super duper! You'll need to make sure you have yout X.509 certificate downloaded too.
+Save this in your ~/.ec2 directory.
+
+Press enter when you're ready
+EOS
+
+            cert_help_str = <<-EOE
+To get your X.509 certificates, navigate to http://aws.amazon.com and login. Click on
+Your Account and scroll down. The X.509 Certificate box is at the bottom. Make sure you
+click download and save the cert-*.pem file. If you don't know or don't have the pk-*.pem file
+you may have to recreate it. Not to worry though, it's super easy. Click on the Create New button. 
+From there, you can download the cert-*.pem and the pk-*.pem files.
+
+            EOE
+
+            ask_with_help :message => rescued_ask_str,
+                          :help => cert_help_str do |t|
+
+begin
+  ec2 = PoolParty::Remote::Ec2.ec2({:access_key => @access_key, :secret_access_key => @secret_access_key})
+  keypairs = ec2.describe_keypairs["keySet"]["item"]
+  keynames = keypairs.map {|k| k["keyName"]}  
+rescue Exception => e
+  colored_say "There was an error: #{e}. Recheck your access_key and secret_access_key to make sure you copied them correctly"
+  exit 1
+end
+
+key_str =<<-EOK
+
+Finally, what's the name of the keypair you'd like to use?
+
+You already have the following keypairs setup:
+\t#{keynames.join("\n\t")}
+
+You can use one of these keys, or create a new one.
+EOK
+
+key_str_help =<<-EOH
+  To make a new keypair, make sure you have the ec2 tools installed. You can create a new one by typing the command:
+    ec2-add-keypair
+  
+  Save this into a file in your ~/.ec2 directory. Enter the name of the keypair here.
+EOH
+            ask_with_help :message => key_str,
+                          :help => key_str_help do |t|
+                @keypair = t
+                say <<-EOE
+
+In your clouds.rb, you can use this keypair in your clouds.
+  
+Create one for each cloud you want to use.
+
+                EOE
+                
+                clds =<<-EOC
+pool :my_pool do
+  cloud :my_app do
+
+    # Copy these lines and source them in your .profile or .bashrc file
+    # export AWS_ACCESS_KEY="#{@access_key}"
+    # export AWS_SECRET_ACCESS_KEY="#{@secret_access_key}"
+    # export EC2_PRIVATE_KEY=$(ls ~/.ec2/pk-*.pem)
+    # export EC2_CERT=$(ls ~/.ec2/cert-*.pem)
+
+    access_key "#{@access_key}"
+    secret_access_key "#{@secret_access_key}"
+    keypair "#{@keypair}"
+    
+    has_file "/etc/motd" do
+      content "Welcome to your first PoolParty instance!"
+    end
+  end
+end             
+                EOC
+                
+                ::File.open("clouds.rb", "w") {|f| f << clds}
+                colored_say closing_message
+              end
+            end            
+            
+          end
+        end
+      when :vmrun
+        vmrun_file_help =<<-EOV
+Vmware uses a vmwarevm file to keep information about the vmware instance. To find the vmwarevm file, 
+navigate to vmware and find the vm you'd like to use. Find this in finder and paste that here.
+        EOV
+        default_vmrun_file = Dir["#{::File.expand_path("~")}/Documents/Virtual\ Machines/*.vmwarevm"].first rescue nil
+        vmrun_file = <<-EOE
+Awesome. What's the path to your vmwarevm file?
+#{
+if default_vmrun_file
+ "Is this it: #{default_vmrun_file}? (Press enter to accept default)" 
+end
+}
+        EOE
+        ask_with_help :message => vmrun_file,
+                      :help => vmrun_file_help do |t|
+          
+          t = default_vmrun_file if t.nil? || t.empty?
+          ip_help =<<-EOV
+Right now, vmrun, the remoter base needs an explicitly set ip. Log into your vm and type ifconfig. Copy and paste that here.
+          EOV
+          ask_with_help :message => "Now, what's the ip of your vm?",
+                        :validate => /\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|h|H/,
+                        :help => ip_help do |i|
+           
+           ask_with_help :message => "Finally, make sure you get your key on the vm. You can use this command: \n\tscp ~/.ssh/id_rsa.pub root@#{i}:/root/.ssh/authorized_keys.\n\nPress enter when you're ready" do |qqqq|
+             
+          vmx_file = Dir["#{t}/*.vmx"].first
+                clds =<<-EOC
+pool :my_pool do
+  cloud :my_app do
+    using :vmrun do
+      vmx_hash({
+        "#{vmx_file}" => "#{i}"
+      })
+    end
+      
+    has_file "/etc/motd" do
+      content "Welcome to your first PoolParty instance!"
+    end
+  end
+end             
+                EOC
+
+                ::File.open("clouds.rb", "w") {|f| f << clds}
+                colored_say closing_message
+                
+              end
+            end
+          end
+      else
+        say("Not sure what remoter_base you mean. Please choose from the menu.")
+      end      
+    end
+  end
+  
+end

data/bin/server-butterfly

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+# based on snippet from ciconia
+# http://snippets.dzone.com/posts/show/2265
+
+require 'fileutils'
+require File.dirname(__FILE__)+'/../vendor/gems/butterfly/lib/butterfly.rb'
+require 'rack'
+
+monitor_dir = ::File.join(::File.dirname(__FILE__),'..','lib/poolparty/monitors/')
+require(::File.join(monitor_dir, 'stats_monitor_adaptor.rb') )
+opts ={ :adaptor_opts => {
+           :file => ::File.join(monitor_dir, 'stats_monitor_adaptor.rb')},
+           :port => PoolParty::Default.butterfly_port
+      }
+
+app = Rack::Builder.new do
+  use Rack::ShowExceptions
+  # use Rack::Lint
+  run Butterfly::Server.new(opts)
+end
+
+# pid = fork do
+  Rack::Handler::Thin.run(app, :Port => PoolParty::Default.butterfly_port)
+# end
+# Process.detach pid

data/bin/server-cloud-elections

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+$:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+require "poolparty"
+require "poolparty/lite"
+
+o = PoolParty::Optioner.new(ARGV) do |opts, optioner|
+  opts.on('-n cloudname', '--name name', 'Start cloud by this name')    { |c| optioner.cloudname c }
+  opts.on('-i num', '--num i', 'Instance number')    { |i| optioner.inst_num i.to_i }
+end
+
+vote_for = o.unflagged_args.shift || "none"
+
+schema = ::PoolParty.load_cloud_from_json(o.unflagged_args.shift)  #TODO: smarter option parser
+remote_base_name = schema.options.remote_base.split('::')[-1].camelcase
+remoter_base = PoolParty::Remote.module_eval(remote_base_name)
+
+o.loaded_clouds.each do |cld|
+  cld.update_from_schema(schema)
+  
+  case vote_for
+  when "expand"
+    vputs "Launching new instance"
+    cld.send :launch_instance!, schema.options do |node|
+      dputs "Started node at #{node[:ip]}"
+      
+      ::PoolParty::Provision::BootStrapper.new( node[:ip], :cloud => cld )
+      ::PoolParty::Provision::DrConfigure.new( node[:ip], :cloud => cld )
+    end
+    
+  when "contract"
+    last_instance_id = %x[/usr/bin/server-list-active instance_id].split("\t").last
+    remoter_base.terminate_instance! schema.options.merge(:instance_id => last_instance_id)
+  else
+    puts "Unknown action: #{vote_for}"
+  end
+
+end

data/bin/server-ensure-provisioning

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+$:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+require "poolparty"
+require "poolparty/lite"
+
+o = PoolParty::Optioner.new(ARGV) do |opts, optioner|
+  opts.on('-n cloudname', '--name name', 'Start cloud by this name')    { |c| optioner.cloudname c }
+  opts.on('-i num', '--num i', 'Instance number')    { |i| optioner.inst_num i.to_i }
+end
+
+schema = ::PoolParty.load_cloud_from_json
+
+o.loaded_clouds.each do |cld|
+  cld.update_from_schema(schema)
+    
+  # If an IP or DNS name is given, bootstrap that node, otherwise, bootstrap all running nodes.
+  nodes = !o.unflagged_args.empty? ?  o.unflagged_args : cld.remote_instances_list.collect {|inst| inst.ip if inst.running? }.compact
+  if nodes.empty?
+    puts "No nodes to bootstrap"
+  else
+    if o.inst_num?
+      ::PoolParty::Provision::BootStrapper.new( nodes[o.inst_num], :cloud => cld )
+      ::PoolParty::Provision::DrConfigure.new( nodes[o.inst_num], :cloud => cld )
+    else
+      nodes.each do |address|
+        puts "bootstrapping: #{address}"
+        ::PoolParty::Provision::BootStrapper.new( address, :cloud => cld )
+        ::PoolParty::Provision::DrConfigure.new( address, :cloud => cld )
+      end
+    end
+  end
+end
+

data/bin/server-get-load

@@ -1,18 +1,15 @@
 #!/usr/bin/env ruby
 $:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
-require "poolparty"
-require "poolpartycl"
+require "poolparty/lite"
+Dir["#{File.dirname(__FILE__)}/../lib/poolparty/monitors/base_monitor.rb"].each {|f| require f}
 
-available_monitors = PoolParty::Monitors.available_monitors
-o = PoolParty::Optioner.new(ARGV) do |opts, optioner|
-  # opts.on('-n name', '--name name', 'Listing for cloud name')    { |c| optioner.cloudname c }
-  opts.on('-m name', '--monitor monitor_name', "Monitor name (default: cpu) (Available monitors: #{available_monitors.join(", ")})") {|c| optioner.monitorname c}
-end
+monitor = ARGV.pop || "cpu"
 
-@monitor = o.monitorname ? o.monitorname : "cpu"
+begin
+  Dir["#{File.dirname(__FILE__)}/../lib/poolparty/monitors/monitors/#{monitor}_monitor.rb"].each {|f| require f}
   
-if PoolParty::Monitors.available_monitors.include?(@monitor.downcase.to_sym)
-  print "PoolParty::Monitors::#{@monitor.capitalize}Monitor".constantize.run.to_f
-else
-  puts "Unknown monitor"
+  klass = eval("PoolParty::Monitors::#{monitor.capitalize}Monitor")
+  print (klass.send :run).to_f rescue -1.0
+rescue Exception => e
+  puts -1.0
 end

data/bin/server-list-active

@@ -1,18 +1,31 @@
 #!/usr/bin/env ruby
 $:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
-require "poolparty"
-require "poolpartycl"
+require "poolparty/lite"
 
-o = PoolParty::Optioner.new(ARGV) do |opts, optioner|
-  opts.on('-c name', '--class name', 'Listing for cloud class (valid args: [name|ip])')    { |t| optioner.classname t }
-  opts.on('-n name', '--name name', 'Listing for cloud name')    { |c| optioner.cloudname c }
+list_type = ARGV.shift || "ip"
+
+json_file = ARGV[0] || PoolParty::Default.properties_hash_file
+raise "ERROR: no json_file given" if !json_file
+schema = PoolParty::Schema.new( ::File.read(json_file) )
+
+inst_array = PoolParty::Neighborhoods.load_default.instances.instances
+
+if !inst_array || inst_array.empty?
+  remoter_base = schema.options.remote_base
+  remote_base = remoter_base.top_level_class.constantize(PoolParty::Remote)
+
+  instances = remote_base.send :describe_instances, schema.options
+  inst_array = instances.map {|inst| inst.send list_type.to_sym }  
 end
 
-@ctype = (o.classname || "name").downcase.to_sym
+print_array = 
+case list_type
+when "ip"
+  inst_array.compact.map {|i| i.ip }
+when "internal_ip"
+  inst_array.compact.map {|i| i.internal_ip || i.ip }
+else
+  inst_array
+end
 
-o.loaded_clouds.each do |cloud|
-  with_cloud(cloud, {:ctype => @ctype}) do
-    print "#{list_of_running_instances.collect{|a| "#{a.send ctype}" }.join("\t")}"
-  end  
-  
-end
+print print_array.compact.join("\t")

data/bin/server-manage-election

@@ -0,0 +1,67 @@
+#!/usr/bin/env ruby
+# Call server-list-active and get the neighborhood ips back
+# then call each of those at the port of butterfly and retain the json
+# From there, mash it up together and call server-handle-election
+# to handle the election process
+
+# Error codes
+# 1 -> clouds.json was not loaded
+# 2 -> server-list-active didn't run
+require "rubygems"
+require 'net/http'
+require 'json'
+require "poolparty/lite"
+
+status = ARGV.shift || "running" # or pending or terminated
+schema = ::PoolParty.load_cloud_from_json(ARGV.shift)  #TODO: smarter option parser
+
+def get_json_stats_from(ip, port)
+  Net::HTTP.start(ip, port) {|http| http.get('/stats.json') }.body rescue "{}"
+end
+ 
+ips = %x[/usr/bin/server-list-active internal_ip].split("\t")
+
+exit(2) unless ips
+
+minimum_instances = schema.options.minimum_instances
+maximum_instances = schema.options.maximum_instances
+
+if ips.size < minimum_instances
+  %x[/usr/bin/server-cloud-elections expand]
+  puts "Cloud is too small, need to expand"
+  exit(0)
+elsif ips.size > maximum_instances
+  %x[/usr/bin/server-cloud-elections contract]
+  puts "Cloud is too big, need to contract"
+  exit(0)
+end
+
+total_json_hash = {}
+ips.each do |ip|
+  node_hsh = JSON.parse(get_json_stats_from(ip, 8642))
+  total_json_hash.merge!(node_hsh) if node_hsh
+end
+
+candidates = {:expand => 0, :contract => 0, :elected_action => nil}
+candidates.each do |action, ballots|
+  total_json_hash.each do |ip, node_hsh|
+    candidates[action]+=1 if node_hsh["nominations"] && node_hsh["nominations"].include?(action.to_s)
+    candidates[:elected_action] = node_hsh["elected_action"]
+  end
+end
+
+puts total_json_hash.inspect
+puts candidates.inspect
+
+# Expand the cloud if 50+% of the votes are for expansion
+# Contract the cloud if 51+% of the votes are for contraction
+@elected_action = candidates[:elected_action] || "none"
+if (candidates[:expand] - candidates[:contract])/total_json_hash.size > 0.5
+  %x[/usr/bin/server-cloud-elections expand] unless @elected_action == "expand"
+  @elected_action = "expand"
+elsif (candidates[:contract] - candidates[:expand])/total_json_hash.size > 0.5
+  %x[/usr/bin/server-cloud-elections contract] unless @elected_action == "contract"
+  @elected_action = "contract"
+end      
+
+puts "Elected action: #{@elected_action}"

data/bin/server-monitor.ru

@@ -0,0 +1,42 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'rack'
+# require 'rack/contrib'
+require 'json'
+require 'thin'
+require ::File.join(::File.dirname(__FILE__),'..','lib/poolparty/monitors/', 'monitor_rack.rb') 
+
+class Baton
+  def initialize(app)
+    @app = app
+  end
+  def call(env)
+    self.class.call(env)
+  end
+  def self.call(env)
+    puts "received: #{env.to_yaml}\n------\n"
+    response = Rack::Response.new
+    response.write "requested: #{env["REQUEST_PATH"]}"
+    response.finish
+  end
+end
+
+
+app = Rack::Builder.new do
+  use Rack::Reloader, 2
+  use Rack::ShowExceptions
+  use Rack::Lint
+  use Baton
+  # use Rack::PostBodyContentTypeParser  #parses json requests to params hash
+  run Monitors::MonitorRack.new()
+  run_after
+end
+
+if __FILE__ == $0
+# at_exit do
+  Rack::Handler::Thin.run app, :daemonize=>true, :pid_file=>'/tmp/monitor.pid'
+  # Thin::Server.new(app, :daemonize=>true, :pid_file=>'/tmp/monitor.pid').daemonize
+# end
+else
+  run app
+end

data/bin/server-query-agent

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+$:.unshift(File.join(File.dirname(__FILE__), "..", "lib", "poolparty"))
+require "socket"
+Dir["#{::File.dirname(__FILE__)}/../lib/poolparty/modules/*.rb"].each {|f| require f }
+require "poolparty/base"
+
+def query(meth, *args)
+  socket = TCPSocket.open("127.0.0.1", PoolParty::Default.agent_port)
+  out = socket.send("#{meth} #{args}", 100)
+  socket.close
+  out
+end
+
+meth = ARGV.shift
+puts query(meth, ARGV)

data/bin/server-rerun

@@ -16,7 +16,7 @@ include Remote
 @clouds.each do |cl|
   
   with_cloud(cl, {:testing => o.testing}) do
-    cmd = PoolParty::Remote::RemoteInstance.puppet_runner_command
+    cmd = PoolParty::Remote::RemoteInstance.dependency_resolver_comand
     %x[#{cmd}] unless testing
   end  
   

data/bin/server-write-new-nodes

@@ -11,11 +11,11 @@ end
 o.loaded_clouds.each do |cloud|
   with_cloud(cloud) do    
 
-    new_nodes_file = "#{Base.manifest_path}/nodes/nodes.pp"
+    new_nodes_file = "#{Default.manifest_path}/nodes/nodes.pp"
             
     str = returning Array.new do |arr|
       arr << "node default { include poolparty }"
-      list_of_running_instances.each do |ri| 
+      nodes(:status => "running").each do |ri| 
         arr << "node \"#{ri.name}\" inherits default {}"
       end
     end.join("\n")