pec2 0.3.2 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bb3c1e15764fe1010d654500b273e83e7bba6ba8
-  data.tar.gz: 36009a1c8f704b665575cd9cdb6a31798beccbc9
+  metadata.gz: fb281220d6b29ff7d3fee0f3f981680855d5cc08
+  data.tar.gz: de60d8338782ebc4214835e90aba81d7171b0959
 SHA512:
-  metadata.gz: 2ef1e5ee199e5a685b4d9755c3b53e87689aa0e14725a4f204aaa05e461a85de905186fb24d71e108611a3b9b627a049deaa95ffeba53db5bb082c288e1aa003
-  data.tar.gz: 40f7f0431d95e39e4aab7d465a1e391df8b03547f861d201b2962b670718ba54ecf2a3ca92236ff0304d770e22a3be99128216837b924dd2a34f69d89837b7e4
+  metadata.gz: '08a9d23eb9c518f2fd8edc23f19fc5f5077a83626545c40331e202e680f98606547e9c9ceb666013b4e92f754459f53a88a9f97a4f804c25ad31f4255996baff'
+  data.tar.gz: b65d6d2e8b3f7f6a395e1cbd9d1a75b47d28eb828a1af7d49a33c9391eabe04ef309ca9bb6e8320822e4cfbbdf8baaf30d7e99da816928dcb968f1bc7ae8a26f

data/.gitignore

@@ -3,6 +3,7 @@ doc/
 pkg/
 tmp/
 **/*.gem
+**/*.pyc
 .yardoc/
 .bundle/
 vendor/bundle/

data/README.md

@@ -4,6 +4,8 @@ Run parallel ssh command for ec2.
 
 Commands can run to multiple hosts at once using the ec2 tag.
 
+required python.
+
 ## Examples
 
     $ bundle exec pec2 -t Project:project_a Stages:production -c 'hostname'
@@ -12,7 +14,7 @@ Commands can run to multiple hosts at once using the ec2 tag.
 
     $ bundle exec pec2 -t Project:project_a Stages:production -c 'sudo hostname' -P -s ${sudo_password}
 
-## Parallel number control
+## Parallel number control(150 threads)
 
     $ bundle exec pec2 -t Project:embulk Stages:production -c 'hostname' -P -p 150
 

data/exe/bin/pnuke

@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+# -*- Mode: python -*-
+
+# Copyright (c) 2009-2012, Andrew McNabb
+# Copyright (c) 2003-2008, Brent N. Chun
+
+"""Nukes all processes that match pattern running as user on the set of nodes
+in hosts.txt.
+"""
+
+import os
+import sys
+
+parent, bindir = os.path.split(os.path.dirname(os.path.abspath(sys.argv[0])))
+if os.path.exists(os.path.join(parent, 'psshlib')):
+    sys.path.insert(0, parent)
+
+from psshlib import psshutil
+from psshlib.task import Task
+from psshlib.manager import Manager, FatalError
+from psshlib.cli import common_parser, common_defaults
+
+_DEFAULT_TIMEOUT = 60
+
+def option_parser():
+    parser = common_parser()
+    parser.usage = "%prog [OPTIONS] pattern"
+    parser.epilog = "Example: pnuke -h hosts.txt -l irb2 java"
+    return parser
+
+def parse_args():
+    parser = option_parser()
+    defaults = common_defaults(timeout=_DEFAULT_TIMEOUT)
+    parser.set_defaults(**defaults)
+    opts, args = parser.parse_args()
+
+    if len(args) < 1:
+        parser.error('Pattern not specified.')
+
+    if len(args) > 1:
+        parser.error('Extra arguments given after the pattern.')
+
+    if not opts.host_files and not opts.host_strings:
+        parser.error('Hosts not specified.')
+
+    return opts, args
+
+def do_pnuke(hosts, pattern, opts):
+    if opts.outdir and not os.path.exists(opts.outdir):
+        os.makedirs(opts.outdir)
+    if opts.errdir and not os.path.exists(opts.errdir):
+        os.makedirs(opts.errdir)
+    manager = Manager(opts)
+    for host, port, user in hosts:
+        cmd = ['ssh', host, '-o', 'NumberOfPasswordPrompts=1']
+        if opts.options:
+            for opt in opts.options:
+                cmd += ['-o', opt]
+        if user:
+            cmd += ['-l', user]
+        if port:
+            cmd += ['-p', port]
+        if opts.extra:
+            cmd.extend(opts.extra)
+        cmd.append('pkill -9 %s' % pattern)
+        t = Task(host, port, user, cmd, opts)
+        manager.add_task(t)
+    try:
+        statuses = manager.run()
+    except FatalError:
+        sys.exit(1)
+
+    if min(statuses) < 0:
+        # At least one process was killed.
+        sys.exit(3)
+    for status in statuses:
+        if status == 255:
+            sys.exit(4)
+    for status in statuses:
+        if status != 0:
+            sys.exit(5)
+
+if __name__ == "__main__":
+    opts, args = parse_args()
+    pattern = args[0]
+    try:
+        hosts = psshutil.read_host_files(opts.host_files,
+            default_user=opts.user)
+    except IOError:
+        _, e, _ = sys.exc_info()
+        sys.stderr.write('Could not open hosts file: %s\n' % e.strerror)
+        sys.exit(1)
+    if opts.host_strings:
+        for s in opts.host_strings:
+            hosts.extend(psshutil.parse_host_string(s, default_user=opts.user))
+    do_pnuke(hosts, pattern, opts)

data/exe/bin/prsync

@@ -0,0 +1,126 @@
+#!/usr/bin/env python
+# -*- Mode: python -*-
+
+# Copyright (c) 2009-2012, Andrew McNabb
+# Copyright (c) 2003-2008, Brent N. Chun
+
+"""Parallel rsync to the set of nodes in hosts.txt.
+
+For each node, we essentially do a rsync -rv -e ssh local user@host:remote.
+Note that remote must be an absolute path.
+"""
+
+import os
+import re
+import sys
+
+parent, bindir = os.path.split(os.path.dirname(os.path.abspath(sys.argv[0])))
+if os.path.exists(os.path.join(parent, 'psshlib')):
+    sys.path.insert(0, parent)
+
+from psshlib import psshutil
+from psshlib.task import Task
+from psshlib.manager import Manager, FatalError
+from psshlib.cli import common_parser, common_defaults
+
+def option_parser():
+    parser = common_parser()
+    parser.usage = "%prog [OPTIONS] local remote"
+    parser.epilog = ("Example: prsync -r -h hosts.txt -l irb2 foo " +
+          "/home/irb2/foo")
+
+    parser.add_option('-r', '--recursive', dest='recursive',
+            action='store_true', help='recusively copy directories (OPTIONAL)')
+    parser.add_option('-a', '--archive', dest='archive', action='store_true',
+            help='use rsync -a (archive mode) (OPTIONAL)')
+    parser.add_option('-z', '--compress', dest='compress', action='store_true',
+            help='use rsync compression (OPTIONAL)')
+    parser.add_option('-S', '--ssh-args', metavar="ARGS", dest='ssh_args',
+            action='store', help='extra arguments for ssh')
+
+    return parser
+
+def parse_args():
+    parser = option_parser()
+    defaults = common_defaults()
+    parser.set_defaults(**defaults)
+    opts, args = parser.parse_args()
+
+    if len(args) < 1:
+        parser.error('Paths not specified.')
+
+    if len(args) < 2:
+        parser.error('Remote path not specified.')
+
+    if len(args) > 2:
+        parser.error('Extra arguments given after the remote path.')
+
+    if not opts.host_files and not opts.host_strings:
+        parser.error('Hosts not specified.')
+
+    return opts, args
+
+def do_prsync(hosts, local, remote, opts):
+    if opts.outdir and not os.path.exists(opts.outdir):
+        os.makedirs(opts.outdir)
+    if opts.errdir and not os.path.exists(opts.errdir):
+        os.makedirs(opts.errdir)
+    manager = Manager(opts)
+    for host, port, user in hosts:
+        ssh = ['ssh']
+        if opts.options:
+            for opt in opts.options:
+                ssh += ['-o', opt]
+        if port:
+            ssh += ['-p', port]
+        if opts.ssh_args:
+            ssh += [opts.ssh_args]
+
+        cmd = ['rsync', '-e', ' '.join(ssh)]
+        if opts.verbose:
+            cmd.append('-v')
+        if opts.recursive:
+            cmd.append('-r')
+        if opts.archive:
+            cmd.append('-a')
+        if opts.compress:
+            cmd.append('-z')
+        if opts.extra:
+            cmd.extend(opts.extra)
+        cmd.append(local)
+        if user:
+            cmd.append('%s@%s:%s' % (user, host, remote))
+        else:
+            cmd.append('%s:%s' % (host, remote))
+        t = Task(host, port, user, cmd, opts)
+        manager.add_task(t)
+    try:
+        statuses = manager.run()
+    except FatalError:
+        sys.exit(1)
+
+    if min(statuses) < 0:
+        # At least one process was killed.
+        sys.exit(3)
+    for status in statuses:
+        if status != 0:
+            sys.exit(4)
+
+if __name__ == "__main__":
+    opts, args = parse_args()
+    local = args[0]
+    remote = args[1]
+    if not re.match("^/", remote):
+        print("Remote path %s must be an absolute path" % remote)
+        sys.exit(3)
+    try:
+        hosts = psshutil.read_host_files(opts.host_files,
+            default_user=opts.user)
+    except IOError:
+        _, e, _ = sys.exc_info()
+        sys.stderr.write('Could not open hosts file: %s\n' % e.strerror)
+        sys.exit(1)
+    if opts.host_strings:
+        for s in opts.host_strings:
+            hosts.extend(psshutil.parse_host_string(s, default_user=opts.user))
+    do_prsync(hosts, local, remote, opts)

data/exe/bin/pscp

@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+# -*- Mode: python -*-
+
+# Copyright (c) 2009-2012, Andrew McNabb
+# Copyright (c) 2003-2008, Brent N. Chun
+
+"""Parallel scp to the set of nodes in hosts.txt.
+
+For each node, we essentially do a scp [-r] local user@host:remote.  This
+program also uses the -q (quiet) and -C (compression) options.  Note that
+remote must be an absolute path.
+"""
+
+import os
+import re
+import sys
+
+parent, bindir = os.path.split(os.path.dirname(os.path.abspath(sys.argv[0])))
+if os.path.exists(os.path.join(parent, 'psshlib')):
+    sys.path.insert(0, parent)
+
+from psshlib import psshutil
+from psshlib.task import Task
+from psshlib.manager import Manager, FatalError
+from psshlib.cli import common_parser, common_defaults
+
+def option_parser():
+    parser = common_parser()
+    parser.usage = "%prog [OPTIONS] local remote"
+    parser.epilog = ("Example: pscp -h hosts.txt -l irb2 foo.txt " +
+            "/home/irb2/foo.txt")
+
+    parser.add_option('-r', '--recursive', dest='recursive',
+            action='store_true', help='recusively copy directories (OPTIONAL)')
+
+    return parser
+
+def parse_args():
+    parser = option_parser()
+    defaults = common_defaults()
+    parser.set_defaults(**defaults)
+    opts, args = parser.parse_args()
+
+    if len(args) < 1:
+        parser.error('Paths not specified.')
+
+    if len(args) < 2:
+        parser.error('Remote path not specified.')
+
+    if not opts.host_files and not opts.host_strings:
+        parser.error('Hosts not specified.')
+
+    return opts, args
+
+def do_pscp(hosts, localargs, remote, opts):
+    if opts.outdir and not os.path.exists(opts.outdir):
+        os.makedirs(opts.outdir)
+    if opts.errdir and not os.path.exists(opts.errdir):
+        os.makedirs(opts.errdir)
+    manager = Manager(opts)
+    for host, port, user in hosts:
+        cmd = ['scp', '-qC']
+        if opts.options:
+            for opt in opts.options:
+                cmd += ['-o', opt]
+        if port:
+            cmd += ['-P', port]
+        if opts.recursive:
+            cmd.append('-r')
+        if opts.extra:
+            cmd.extend(opts.extra)
+        cmd.extend(localargs)
+        if user:
+            cmd.append('%s@%s:%s' % (user, host, remote))
+        else:
+            cmd.append('%s:%s' % (host, remote))
+        t = Task(host, port, user, cmd, opts)
+        manager.add_task(t)
+    try:
+        statuses = manager.run()
+    except FatalError:
+        sys.exit(1)
+
+    if min(statuses) < 0:
+        # At least one process was killed.
+        sys.exit(3)
+    for status in statuses:
+        if status != 0:
+            sys.exit(4)
+
+if __name__ == "__main__":
+    opts, args = parse_args()
+    localargs = args[0:-1]
+    remote = args[-1]
+    if not re.match("^/", remote):
+        print("Remote path %s must be an absolute path" % remote)
+        sys.exit(3)
+    try:
+        hosts = psshutil.read_host_files(opts.host_files,
+            default_user=opts.user)
+    except IOError:
+        _, e, _ = sys.exc_info()
+        sys.stderr.write('Could not open hosts file: %s\n' % e.strerror)
+        sys.exit(1)
+    if opts.host_strings:
+        for s in opts.host_strings:
+            hosts.extend(psshutil.parse_host_string(s, default_user=opts.user))
+    do_pscp(hosts, localargs, remote, opts)

data/exe/bin/pslurp

@@ -0,0 +1,129 @@
+#!/usr/bin/env python
+# -*- Mode: python -*-
+
+# Copyright (c) 2009-2012, Andrew McNabb
+# Copyright (c) 2003-2008, Brent N. Chun
+
+"""Parallel scp from the set of nodes in hosts.txt.
+
+For each node, we essentially do a scp [-r] user@host:remote
+outdir/<node>/local.  This program also uses the -q (quiet) and -C
+(compression) options.  Note that remote must be an absolute path.
+"""
+
+import os
+import re
+import sys
+
+parent, bindir = os.path.split(os.path.dirname(os.path.abspath(sys.argv[0])))
+if os.path.exists(os.path.join(parent, 'psshlib')):
+    sys.path.insert(0, parent)
+
+from psshlib import psshutil
+from psshlib.task import Task
+from psshlib.manager import Manager, FatalError
+from psshlib.cli import common_parser, common_defaults
+
+def option_parser():
+    parser = common_parser()
+    parser.usage = "%prog [OPTIONS] remote local"
+    parser.epilog = ("Example: pslurp -h hosts.txt -L /tmp/outdir -l irb2 " +
+            "         /home/irb2/foo.txt foo.txt")
+
+    parser.add_option('-r', '--recursive', dest='recursive',
+            action='store_true', help='recusively copy directories (OPTIONAL)')
+    parser.add_option('-L', '--localdir', dest='localdir',
+            help='output directory for remote file copies')
+
+    return parser
+
+def parse_args():
+    parser = option_parser()
+    defaults = common_defaults()
+    parser.set_defaults(**defaults)
+    opts, args = parser.parse_args()
+
+    if len(args) < 1:
+        parser.error('Paths not specified.')
+
+    if len(args) < 2:
+        parser.error('Local path not specified.')
+
+    if len(args) > 2:
+        parser.error('Extra arguments given after the local path.')
+
+    if not opts.host_files and not opts.host_strings:
+        parser.error('Hosts not specified.')
+
+    return opts, args
+
+def do_pslurp(hosts, remote, local, opts):
+    if opts.localdir and not os.path.exists(opts.localdir):
+        os.makedirs(opts.localdir)
+    if opts.outdir and not os.path.exists(opts.outdir):
+        os.makedirs(opts.outdir)
+    if opts.errdir and not os.path.exists(opts.errdir):
+        os.makedirs(opts.errdir)
+    for host, port, user in hosts:
+        if opts.localdir:
+            dirname = "%s/%s" % (opts.localdir, host)
+        else:
+            dirname = host
+        if not os.path.exists(dirname):
+            os.mkdir(dirname)
+    manager = Manager(opts)
+    for host, port, user in hosts:
+        if opts.localdir:
+            localpath = "%s/%s/%s" % (opts.localdir, host, local)
+        else:
+            localpath = "%s/%s" % (host, local)
+        cmd = ['scp', '-qC']
+        if opts.options:
+            for opt in opts.options:
+                cmd += ['-o', opt]
+        if port:
+            cmd += ['-P', port]
+        if opts.recursive:
+            cmd.append('-r')
+        if opts.extra:
+            cmd.extend(opts.extra)
+        if user:
+            cmd.append('%s@%s:%s' % (user, host, remote))
+        else:
+            cmd.append('%s:%s' % (host, remote))
+        cmd.append(localpath)
+        t = Task(host, port, user, cmd, opts)
+        manager.add_task(t)
+    try:
+        statuses = manager.run()
+    except FatalError:
+        sys.exit(1)
+
+    if min(statuses) < 0:
+        # At least one process was killed.
+        sys.exit(3)
+    for status in statuses:
+        if status == 255:
+            sys.exit(4)
+    for status in statuses:
+        if status != 0:
+            sys.exit(5)
+
+if __name__ == "__main__":
+    opts, args = parse_args()
+    remote = args[0]
+    local = args[1]
+    if not re.match("^/", remote):
+        print("Remote path %s must be an absolute path" % remote)
+        sys.exit(3)
+    try:
+        hosts = psshutil.read_host_files(opts.host_files,
+            default_user=opts.user)
+    except IOError:
+        _, e, _ = sys.exc_info()
+        sys.stderr.write('Could not open hosts file: %s\n' % e.strerror)
+        sys.exit(1)
+    if opts.host_strings:
+        for s in opts.host_strings:
+            hosts.extend(psshutil.parse_host_string(s, default_user=opts.user))
+    do_pslurp(hosts, remote, local, opts)