stellar_core_commander 0.0.11 → 0.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 39b1d602547d8c0fb71b64ee13f913d33710d259
-  data.tar.gz: 153ed88d97f786ed976fe2094aa0857110179719
+  metadata.gz: 194aa47932a7b30b7b5f890cf6a7d7d49144438b
+  data.tar.gz: 000cd0e89691d63bf4c29d7f506bed45180023f1
 SHA512:
-  metadata.gz: 522bf96970380a8cd175ff844f3e816051c28dcfbcf20451774c5ea035b579c5b516684eabd4c30443c70646792f8f35680645c9e533c2da99ae9984e61e6dd9
-  data.tar.gz: b70f74d24878e0e6285e324154d1c03ffc34fcf6b597e1814e835dc177803a89c86fc8f2be404b7859d9bfe1acfad47a4b5e3edb33202e3275790f8bc0e47ba9
+  metadata.gz: bec64f9159656b056b23731f0733efbfb3f6b4d563f2937db902435ccf58074788d47a1b7871a3bda94fc2926e084cf576726d229867804450f5aaf99e63bde8
+  data.tar.gz: 6683025f7d1c4aa3081d9336701f3411d09e424cd885d1bd208f962a06ff2a886af6067842d154a27bb0cf0067ed01d85dc82ad2bc7d86f26097ce623c3e75f6

data/.gitignore

@@ -14,3 +14,4 @@
 mkmf.log
 out.sql
 .DS_Store
+*.swp

data/CONTRIBUTING.md

@@ -0,0 +1,60 @@
+# How to contribute
+
+Your contributions to the Stellar network will help improve the world’s financial
+infrastructure, faster.
+
+We want to make it as easy as possible to contribute changes that
+help the Stellar network grow and thrive. There are a few guidelines that we
+ask contributors to follow so that we can merge your changes quickly.
+
+## Getting Started
+
+* Create a GitHub issue for your contribution, assuming one does not already exist.
+  * Clearly describe the issue including steps to reproduce if it is a bug.
+* Fork the repository on GitHub
+
+## Making Changes
+
+* Create a topic branch from where you want to base your work.
+  * This is usually the master branch.
+  * Please avoid working directly on the `master` branch.
+* Make sure you have added the necessary tests for your changes, and make sure all tests pass.
+
+## Submitting Changes
+
+* <a href="https://docs.google.com/forms/d/1g7EF6PERciwn7zfmfke5Sir2n10yddGGSXyZsq98tVY/viewform?usp=send_form">Sign the Contributor License Agreement</a>.
+* All content, comments, and pull requests must follow the [Stellar Community Guidelines](https://www.stellar.org/community-guidelines/). 
+* Push your changes to a topic branch in your fork of the repository.
+* Submit a pull request to the [stellar_core_commander repository](https://github.com/stellar/stellar-tutorials) in the Stellar organization.
+ * Include a descriptive [commit message](https://github.com/erlang/otp/wiki/Writing-good-commit-messages).
+ * Changes contributed via pull request should focus on a single issue at a time.
+ * Rebase your local changes against the master branch. Resolve any conflicts that arise.
+ 
+At this point you're waiting on us. We like to at least comment on pull requests within three 
+business days (and, typically, one business day). We may suggest some changes or improvements or alternatives.
+
+## Making Trivial Changes
+
+### Documentation
+For changes of a trivial nature to comments and documentation, it is not
+always necessary to create a new GitHub issue. In this case, it is
+appropriate to start the first line of a commit with 'doc' instead of
+an issue number. 
+
+# Additional Resources
+
+* [Bug tracker (Github)](https://github.com/stellar/stellar_core_commander/issues)
+* <a href="https://docs.google.com/forms/d/1g7EF6PERciwn7zfmfke5Sir2n10yddGGSXyZsq98tVY/viewform?usp=send_form">Contributor License Agreement</a>
+* [Explore the API](http://docs.stellarhorizon.apiary.io/)
+* [Readme for stellar_core_commander](https://github.com/stellar/stellar_core_commander/blob/master/README.md)
+* #stellar-dev IRC channel on freenode.org
+* #dev channel on [Slack](http://slack.stellar.org)
+
+
+This document is inspired by:
+
+https://github.com/puppetlabs/puppet/blob/master/CONTRIBUTING.md 
+
+https://github.com/thoughtbot/factory_girl_rails/blob/master/CONTRIBUTING.md 
+
+https://github.com/rust-lang/rust/blob/master/CONTRIBUTING.md

data/bin/scc

@@ -1,6 +1,7 @@
 #!/usr/bin/env ruby
 
 require 'stellar_core_commander'
+require 'stellar-base'
 require 'slop'
 
 def run
@@ -19,8 +20,51 @@ def run
       default: 'local'
     on 'w', 'wait',
       'wait for TERM signal before shutting down and cleaning up',
-       argument: false,
-       default: false
+      argument: false,
+      default: false
+    on 'docker-core-image',
+      'docker image to use for stellar-core',
+      argument: true,
+      default: 'stellar/stellar-core'
+    on 'docker-state-image',
+      'docker image to use for state',
+      argument: true,
+      default: 'stellar/stellar-core-state'
+    on 'docker-pull',
+      'docker pull images before use',
+      default: false
+    on 'atlas',
+      'atlas endpoint for publishing metrics (e.g., http://192.168.59.103:7101/api/v1/publish)',
+      argument: true,
+      default: nil
+    on 'atlas-interval',
+      'number of seconds to wait between publishing metric payloads',
+      argument: true,
+      default: 1
+    on 'destination',
+      'where to store logs and other artifacts (default: a temporary directory)',
+      argument: true,
+      default: nil
+    on 'use-s3',
+      'share history via s3',
+      argument: false,
+      default: false
+    on 's3-history-prefix',
+      's3 prefix to store temp history in (default: subdir of s3://history-stg.stellar.org/dev/scc)',
+      argument: true,
+      default: "s3://history-stg.stellar.org/dev/scc/#{Time.now.to_i}-#{rand 100000}"
+    on 's3-history-region',
+      's3 region to store temp history in (default: eu-west-1)',
+      argument: true,
+      default: "eu-west-1"
+    on 'dump-root-db',
+      'when true, dump to root processes sql database after recipe completion',
+      argument: false,
+      default: false
+    on 'network-passphrase',
+      'sets the network passphrase to configure on all running processes',
+      argument: true,
+      default: Stellar::Networks::TESTNET
   end
 
   recipe    = load_recipe
@@ -29,14 +73,16 @@ def run
   #run recipe
   transactor = StellarCoreCommander::Transactor.new(commander)
 
-
   transactor.run_recipe recipe
   transactor.close_ledger
 
-  output_results(commander.get_root_process transactor)
+  if $opts[:"dump-root-db"]
+    file = commander.get_root_process(transactor).dump_database
+    $stdout.puts IO.read(file)
+  end
 
   if $opts[:wait]
-    puts "Waiting for INT signal..."
+    $stderr.puts "Waiting for INT signal..."
     Signal.trap("INT"){ exit }
     sleep
   end
@@ -46,10 +92,27 @@ end
 def make_commander
   opts = {
     stellar_core_bin: $opts[:"stellar-core-bin"],
+    docker_core_image: $opts[:"docker-core-image"],
+    docker_state_image: $opts[:"docker-state-image"],
+    docker_pull: $opts[:"docker-pull"],
+    atlas: $opts[:"atlas"],
+    atlas_interval: $opts[:"atlas-interval"].to_i,
+    use_s3: $opts[:"use-s3"],
+    s3_history_region: $opts[:"s3-history-region"],
+    s3_history_prefix: $opts[:"s3-history-prefix"],
+    network_passphrase: $opts[:"network-passphrase"],
   }
 
-  StellarCoreCommander::Commander.new($opts[:"process"], opts).tap do |c|
-    c.cleanup_at_exit!
+  destination = $opts[:"destination"]
+  if destination
+    clean_up_destination = false
+  else
+    destination = Dir.mktmpdir("scc")
+    clean_up_destination = true
+  end
+
+  StellarCoreCommander::Commander.new($opts[:"process"], destination, opts).tap do |c|
+    c.cleanup_at_exit!(clean_up_destination)
   end
 end
 
@@ -69,8 +132,4 @@ def load_recipe
   recipe
 end
 
-def output_results(process)
-  $stdout.puts process.dump_database
-end
-
 run

data/examples/cross_host_simple_payment.rb

@@ -1,6 +1,6 @@
-process :node1, [:node1, :node2, :node3], 2, host: '192.168.99.105'
-process :node2, [:node1, :node2, :node3], 2, host: '192.168.99.104'
-process :node3, [:node1, :node2, :node3], 2, host: '192.168.99.103'
+process :node1, [:node1, :node2, :node3], host: '192.168.99.105'
+process :node2, [:node1, :node2, :node3], host: '192.168.99.104'
+process :node3, [:node1, :node2, :node3], host: '192.168.99.103'
 
 account :alice
 account :bob
@@ -11,9 +11,9 @@ on :node1 do
   create_account :alice, :master
   create_account :bob, :master
   close_ledger
-  payment :master, :bob, [:native, 1000_000000]
+  payment :master, :bob, [:native, 100 * Stellar::ONE]
   close_ledger
 end
 
-payment :master, :alice, [:native, 1000_000000]
+payment :master, :alice, [:native, 100 * Stellar::ONE]
 

data/examples/history_generate_and_catchup.rb

@@ -0,0 +1,26 @@
+process :node1, [:node1], accelerate_time: true
+on :node1 do
+  generate_load_and_await_completion 100, 100, 20
+  retry_until_true retries: 100 do
+    ledger_num > 10
+  end
+  check_no_error_metrics
+  check_database_against_ledger_buckets
+end
+
+process :node2_minimal, [:node1], forcescp: false, accelerate_time: true
+on :node2_minimal do
+  retry_until_true retries: 100 do
+    ledger_num > 15
+  end
+  check_integrity_against :node1
+end
+
+
+process :node2_complete, [:node1], forcescp: false, accelerate_time: true, catchup_complete: true
+on :node2_complete do
+  retry_until_true retries: 100 do
+    ledger_num > 15
+  end
+  check_integrity_against :node1
+end

data/examples/history_testnet_catchup.rb

@@ -0,0 +1,15 @@
+process :node1, [:testnet1, :testnet2, :testnet3], forcescp: false, debug: true, validate: false
+on :node1 do
+  raise "node1 synced but failed to catch up" if ledger_num < 5
+  $stderr.puts "caught up on node1: ledger #{ledger_num}"
+  check_no_error_metrics
+  check_database_against_ledger_buckets
+end
+
+process :node2, [:testnet1, :testnet2, :testnet3], forcescp: false, catchup_complete: true, debug: true, validate: false
+on :node2 do
+  raise "node2 synced but failed to catch up" if ledger_num < 5
+  $stderr.puts "caught up on node2: ledger #{ledger_num}"
+  check_no_error_metrics
+  check_database_against_ledger_buckets
+end

data/examples/inflation.rb

@@ -0,0 +1,8 @@
+account :scott
+
+create_account :scott, :master, 2_000_000_000 * Stellar::ONE
+
+close_ledger
+
+set_inflation_dest :scott, :scott
+inflation

data/examples/load_generation.rb

@@ -0,0 +1,15 @@
+process :node1, [:node1, :node2, :node3]
+process :node2, [:node1, :node2, :node3]
+process :node3, [:node1, :node2, :node3]
+
+on :node1 do
+  generate_load_and_await_completion 1000, 1000, 30
+end
+
+on :node2 do
+  check_integrity_against :node1
+end
+
+on :node3 do
+  check_integrity_against :node1
+end

data/examples/load_generation_auto.rb

@@ -0,0 +1,15 @@
+process :node1, [:node1, :node2, :node3]
+process :node2, [:node1, :node2, :node3]
+process :node3, [:node1, :node2, :node3]
+
+on :node1 do
+  generate_load_and_await_completion 10000, 10000, :auto
+end
+
+on :node2 do
+  check_integrity_against :node1
+end
+
+on :node3 do
+  check_integrity_against :node1
+end

data/examples/multi_host_simple_payment.rb

@@ -1,6 +1,6 @@
-process :node1, [:node1, :node2, :node3], 2
-process :node2, [:node1, :node2, :node3], 2
-process :node3, [:node1, :node2, :node3], 2
+process :node1, [:node1, :node2, :node3]
+process :node2, [:node1, :node2, :node3]
+process :node3, [:node1, :node2, :node3]
 
 account :alice
 account :bob
@@ -11,9 +11,16 @@ on :node1 do
   create_account :alice, :master
   create_account :bob, :master
   close_ledger
-  payment :master, :bob, [:native, 1000_000000]
+  payment :master, :bob, [:native, 100 * Stellar::ONE]
   close_ledger
 end
 
-payment :master, :alice, [:native, 1000_000000]
+payment :master, :alice, [:native, 100 * Stellar::ONE]
 
+on :node2 do
+  check_integrity_against :node1
+end
+
+on :node3 do
+  check_integrity_against :node1
+end

data/examples/set_options.rb

@@ -0,0 +1,23 @@
+use_manual_close
+
+account :scott
+account :bartek
+create_account :scott
+create_account :bartek
+
+close_ledger
+
+kp = Stellar::KeyPair.random
+
+set_inflation_dest :scott, :bartek
+set_flags :scott, [:auth_required_flag]
+set_master_signer_weight :scott, 2
+set_thresholds :scott, low: 0, medium: 2, high: 2
+set_thresholds :scott, high: 1
+set_home_domain :scott, "nullstyle.com"
+add_signer :scott, kp, 1
+
+close_ledger
+
+clear_flags :scott, [:auth_required_flag]
+remove_signer :scott, kp

data/examples/simple_payment.rb

@@ -4,4 +4,6 @@ create_account :scott, :master
 
 close_ledger
 
-payment :master, :scott, [:native, 1000 * Stellar::ONE]
+payment :master, :scott, [:native, 1000]
+
+check_no_error_metrics

data/examples/version_mix_consensus.rb

@@ -0,0 +1,46 @@
+old_image = ENV["OLD_IMAGE"]
+new_image = ENV["NEW_IMAGE"]
+
+raise "missing ENV['OLD_IMAGE']" unless old_image
+raise "missing ENV['NEW_IMAGE']" unless new_image
+
+peers = [:oldnode1, :oldnode2, :newnode1, :newnode2]
+
+process :oldnode1, peers, docker_core_image: old_image, docker_pull: true
+process :oldnode2, peers, docker_core_image: old_image, docker_pull: true
+process :newnode1, peers, docker_core_image: new_image, docker_pull: true
+process :newnode2, peers, docker_core_image: new_image, docker_pull: true
+
+account :alice
+account :bob
+
+on :oldnode2 do
+  create_account :alice, :master
+  create_account :bob, :master
+  while not ((account_created :bob) and (account_created :alice))
+    $stderr.puts "Awaiting account-creation"
+    close_ledger
+  end
+  $stderr.puts "oldnode2 bob balance: #{(balance :bob)}"
+  $stderr.puts "oldnode2 alice balance: #{(balance :alice)}"
+  payment :master, :bob, [:native, 100 * Stellar::ONE]
+  close_ledger
+end
+
+on :newnode1 do
+  $stderr.puts "newnode1 bob balance: #{(balance :bob)}"
+  $stderr.puts "newnode1 alice balance: #{(balance :alice)}"
+  raise if (balance :bob) != 1100 * Stellar::ONE
+  payment :master, :alice, [:native, 100 * Stellar::ONE]
+  close_ledger
+  check_integrity_against :oldnode1
+  check_integrity_against :oldnode2
+end
+
+on :oldnode1 do
+  $stderr.puts "oldnode1 bob balance: #{(balance :bob)}"
+  $stderr.puts "oldnode1 alice balance: #{(balance :alice)}"
+  raise if (balance :alice) != 1100 * Stellar::ONE
+  check_integrity_against :newnode1
+  check_integrity_against :newnode2
+end

data/lib/stellar_core_commander.rb

@@ -8,6 +8,7 @@ require "fileutils"
 require "open3"
 require "sequel"
 require "pg"
+require "uri"
 
 module StellarCoreCommander
   extend ActiveSupport::Autoload

data/lib/stellar_core_commander/commander.rb

@@ -1,4 +1,5 @@
 require 'fileutils'
+
 module StellarCoreCommander
 
   #
@@ -8,32 +9,35 @@ module StellarCoreCommander
   class Commander
     include Contracts
 
+    attr_reader :process_options
+
     #
     # Creates a new core commander
     #
-    Contract Or["local", "docker"], Hash => Any
-    def initialize(process_type, process_options={})
+    Contract Or["local", "docker"], String, Hash => Any
+    def initialize(process_type, destination, process_options={})
       @process_type = process_type
+      @destination = destination
       @process_options = process_options
       @processes = []
     end
 
-    Contract Transactor, Symbol, ArrayOf[Symbol], Num, Hash => Process
+    Contract Transactor, Symbol, ArrayOf[Symbol], Hash => Process
     #
     # make_process returns a new, unlaunched Process object, bound to a new
     # tmpdir
-    def make_process(transactor, name, quorum, thresh, options={})
-      tmpdir = Dir.mktmpdir("scc")
+    def make_process(transactor, name, quorum, options={})
+      working_dir = File.join(@destination, name.to_s)
+      FileUtils.mkpath(working_dir)
 
       process_options = @process_options.merge({
         transactor:   transactor,
-        working_dir:  tmpdir,
+        working_dir:  working_dir,
         name:         name,
-        base_port:    39132 + @processes.map(&:required_ports).sum,
+        base_port:    11625 + @processes.map(&:required_ports).sum,
         identity:     Stellar::KeyPair.random,
         quorum:       quorum,
-        threshold:    thresh,
-        manual_close: false
+        manual_close: transactor.manual_close
       }).merge(options)
 
       process_class = case @process_type
@@ -53,32 +57,62 @@ module StellarCoreCommander
     Contract Transactor => Process
     def get_root_process(transactor)
       if @processes.size == 0
-        make_process transactor, :node0, [:node0], 1, { manual_close: transactor.manual_close }
+        make_process transactor, :node0, [:node0]
       end
       @processes[0]
     end
 
     Contract None => ArrayOf[Process]
     def start_all_processes
+      stopped = @processes.select(&:stopped?)
+
+      stopped.each(&:prepare)
+
+      stopped.each do |p|
+        next if p.running?
+
+        $stderr.puts "running #{p.idname} (dir:#{p.working_dir})"
+        p.run
+        p.wait_for_ready if p.await_sync?
+
+      end
+    end
+
+    Contract None => ArrayOf[Process]
+    def require_processes_in_sync
       @processes.each do |p|
-        if not p.running?
-          $stderr.puts "running #{p.idname} (dir:#{p.working_dir})"
-          p.run
-          p.wait_for_ready
+        next unless p.await_sync?
+        begin
+          p.wait_for_ready unless p.synced?
+        rescue Timeout::Error
+          @processes.each do |p2|
+            p2.dump_scp_state
+            p2.dump_info
+            p2.dump_metrics
+            raise "process #{p.name} lost sync"
+          end
         end
       end
     end
 
+    Contract None => Bool
+    def check_no_process_error_metrics
+      @processes.each do |p|
+        p.check_no_error_metrics
+      end
+      true
+    end
+
     def cleanup
       @processes.each(&:cleanup)
     end
 
-    def cleanup_at_exit!
+    def cleanup_at_exit!(clean_up_destination)
       at_exit do
         $stderr.puts "cleaning up #{@processes.length} processes"
         cleanup
+        FileUtils.rm_rf @destination if clean_up_destination
       end
     end
-
   end
 end