gcloud 0.0.4 → 0.0.6

data/packages/gsutil/boto/tests/unit/emr/test_emr_responses.py

@@ -0,0 +1,373 @@
+# Copyright (c) 2010 Jeremy Thurgood <firxen+boto@gmail.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+
+# NOTE: These tests only cover the very simple cases I needed to test
+# for the InstanceGroup fix.
+
+import xml.sax
+import unittest
+
+from boto import handler
+from boto.emr import emrobject
+from boto.resultset import ResultSet
+
+
+JOB_FLOW_EXAMPLE = """
+<DescribeJobFlowsResponse
+    xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-01-15">
+  <DescribeJobFlowsResult>
+    <JobFlows>
+      <member>
+        <ExecutionStatusDetail>
+          <CreationDateTime>2009-01-28T21:49:16Z</CreationDateTime>
+          <StartDateTime>2009-01-28T21:49:16Z</StartDateTime>
+          <State>STARTING</State>
+        </ExecutionStatusDetail>
+        <Name>MyJobFlowName</Name>
+        <LogUri>mybucket/subdir/</LogUri>
+        <Steps>
+          <member>
+            <ExecutionStatusDetail>
+              <CreationDateTime>2009-01-28T21:49:16Z</CreationDateTime>
+              <State>PENDING</State>
+            </ExecutionStatusDetail>
+            <StepConfig>
+              <HadoopJarStep>
+                <Jar>MyJarFile</Jar>
+                <MainClass>MyMailClass</MainClass>
+                <Args>
+                  <member>arg1</member>
+                  <member>arg2</member>
+                </Args>
+                <Properties/>
+              </HadoopJarStep>
+              <Name>MyStepName</Name>
+              <ActionOnFailure>CONTINUE</ActionOnFailure>
+            </StepConfig>
+          </member>
+        </Steps>
+        <JobFlowId>j-3UN6WX5RRO2AG</JobFlowId>
+        <Instances>
+          <Placement>
+            <AvailabilityZone>us-east-1a</AvailabilityZone>
+          </Placement>
+          <SlaveInstanceType>m1.small</SlaveInstanceType>
+          <MasterInstanceType>m1.small</MasterInstanceType>
+          <Ec2KeyName>myec2keyname</Ec2KeyName>
+          <InstanceCount>4</InstanceCount>
+          <KeepJobFlowAliveWhenNoSteps>true</KeepJobFlowAliveWhenNoSteps>
+        </Instances>
+      </member>
+    </JobFlows>
+  </DescribeJobFlowsResult>
+  <ResponseMetadata>
+    <RequestId>9cea3229-ed85-11dd-9877-6fad448a8419</RequestId>
+  </ResponseMetadata>
+</DescribeJobFlowsResponse>
+"""
+
+JOB_FLOW_COMPLETED = """
+<DescribeJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
+  <DescribeJobFlowsResult>
+    <JobFlows>
+      <member>
+        <ExecutionStatusDetail>
+          <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>
+          <LastStateChangeReason>Steps completed</LastStateChangeReason>
+          <StartDateTime>2010-10-21T01:03:59Z</StartDateTime>
+          <ReadyDateTime>2010-10-21T01:03:59Z</ReadyDateTime>
+          <State>COMPLETED</State>
+          <EndDateTime>2010-10-21T01:44:18Z</EndDateTime>
+        </ExecutionStatusDetail>
+        <BootstrapActions/>
+        <Name>RealJobFlowName</Name>
+        <LogUri>s3n://example.emrtest.scripts/jobflow_logs/</LogUri>
+        <Steps>
+          <member>
+            <StepConfig>
+              <HadoopJarStep>
+                <Jar>s3n://us-east-1.elasticmapreduce/libs/script-runner/script-runner.jar</Jar>
+                <Args>
+                  <member>s3n://us-east-1.elasticmapreduce/libs/state-pusher/0.1/fetch</member>
+                </Args>
+                <Properties/>
+              </HadoopJarStep>
+              <Name>Setup Hadoop Debugging</Name>
+              <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure>
+            </StepConfig>
+            <ExecutionStatusDetail>
+              <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>
+              <StartDateTime>2010-10-21T01:03:59Z</StartDateTime>
+              <State>COMPLETED</State>
+              <EndDateTime>2010-10-21T01:04:22Z</EndDateTime>
+            </ExecutionStatusDetail>
+          </member>
+          <member>
+            <StepConfig>
+              <HadoopJarStep>
+                <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar>
+                <Args>
+                  <member>-mapper</member>
+                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-InitialMapper.py</member>
+                  <member>-reducer</member>
+                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-InitialReducer.py</member>
+                  <member>-input</member>
+                  <member>s3://example.emrtest.data/raw/2010/10/20/*</member>
+                  <member>-input</member>
+                  <member>s3://example.emrtest.data/raw/2010/10/19/*</member>
+                  <member>-input</member>
+                  <member>s3://example.emrtest.data/raw/2010/10/18/*</member>
+                  <member>-input</member>
+                  <member>s3://example.emrtest.data/raw/2010/10/17/*</member>
+                  <member>-input</member>
+                  <member>s3://example.emrtest.data/raw/2010/10/16/*</member>
+                  <member>-input</member>
+                  <member>s3://example.emrtest.data/raw/2010/10/15/*</member>
+                  <member>-input</member>
+                  <member>s3://example.emrtest.data/raw/2010/10/14/*</member>
+                  <member>-output</member>
+                  <member>s3://example.emrtest.crunched/</member>
+                </Args>
+                <Properties/>
+              </HadoopJarStep>
+              <Name>testjob_Initial</Name>
+              <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure>
+            </StepConfig>
+            <ExecutionStatusDetail>
+              <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>
+              <StartDateTime>2010-10-21T01:04:22Z</StartDateTime>
+              <State>COMPLETED</State>
+              <EndDateTime>2010-10-21T01:36:18Z</EndDateTime>
+            </ExecutionStatusDetail>
+          </member>
+          <member>
+            <StepConfig>
+              <HadoopJarStep>
+                <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar>
+                <Args>
+                  <member>-mapper</member>
+                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step1Mapper.py</member>
+                  <member>-reducer</member>
+                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step1Reducer.py</member>
+                  <member>-input</member>
+                  <member>s3://example.emrtest.crunched/*</member>
+                  <member>-output</member>
+                  <member>s3://example.emrtest.step1/</member>
+                </Args>
+                <Properties/>
+              </HadoopJarStep>
+              <Name>testjob_step1</Name>
+              <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure>
+            </StepConfig>
+            <ExecutionStatusDetail>
+              <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>
+              <StartDateTime>2010-10-21T01:36:18Z</StartDateTime>
+              <State>COMPLETED</State>
+              <EndDateTime>2010-10-21T01:37:51Z</EndDateTime>
+            </ExecutionStatusDetail>
+          </member>
+          <member>
+            <StepConfig>
+              <HadoopJarStep>
+                <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar>
+                <Args>
+                  <member>-mapper</member>
+                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step2Mapper.py</member>
+                  <member>-reducer</member>
+                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step2Reducer.py</member>
+                  <member>-input</member>
+                  <member>s3://example.emrtest.crunched/*</member>
+                  <member>-output</member>
+                  <member>s3://example.emrtest.step2/</member>
+                </Args>
+                <Properties/>
+              </HadoopJarStep>
+              <Name>testjob_step2</Name>
+              <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure>
+            </StepConfig>
+            <ExecutionStatusDetail>
+              <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>
+              <StartDateTime>2010-10-21T01:37:51Z</StartDateTime>
+              <State>COMPLETED</State>
+              <EndDateTime>2010-10-21T01:39:32Z</EndDateTime>
+            </ExecutionStatusDetail>
+          </member>
+          <member>
+            <StepConfig>
+              <HadoopJarStep>
+                <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar>
+                <Args>
+                  <member>-mapper</member>
+                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step3Mapper.py</member>
+                  <member>-reducer</member>
+                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step3Reducer.py</member>
+                  <member>-input</member>
+                  <member>s3://example.emrtest.step1/*</member>
+                  <member>-output</member>
+                  <member>s3://example.emrtest.step3/</member>
+                </Args>
+                <Properties/>
+              </HadoopJarStep>
+              <Name>testjob_step3</Name>
+              <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure>
+            </StepConfig>
+            <ExecutionStatusDetail>
+              <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>
+              <StartDateTime>2010-10-21T01:39:32Z</StartDateTime>
+              <State>COMPLETED</State>
+              <EndDateTime>2010-10-21T01:41:22Z</EndDateTime>
+            </ExecutionStatusDetail>
+          </member>
+          <member>
+            <StepConfig>
+              <HadoopJarStep>
+                <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar>
+                <Args>
+                  <member>-mapper</member>
+                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step4Mapper.py</member>
+                  <member>-reducer</member>
+                  <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step4Reducer.py</member>
+                  <member>-input</member>
+                  <member>s3://example.emrtest.step1/*</member>
+                  <member>-output</member>
+                  <member>s3://example.emrtest.step4/</member>
+                </Args>
+                <Properties/>
+              </HadoopJarStep>
+              <Name>testjob_step4</Name>
+              <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure>
+            </StepConfig>
+            <ExecutionStatusDetail>
+              <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>
+              <StartDateTime>2010-10-21T01:41:22Z</StartDateTime>
+              <State>COMPLETED</State>
+              <EndDateTime>2010-10-21T01:43:03Z</EndDateTime>
+            </ExecutionStatusDetail>
+          </member>
+        </Steps>
+        <JobFlowId>j-3H3Q13JPFLU22</JobFlowId>
+        <Instances>
+          <SlaveInstanceType>m1.large</SlaveInstanceType>
+          <MasterInstanceId>i-64c21609</MasterInstanceId>
+          <Placement>
+            <AvailabilityZone>us-east-1b</AvailabilityZone>
+          </Placement>
+          <InstanceGroups>
+            <member>
+              <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>
+              <InstanceRunningCount>0</InstanceRunningCount>
+              <StartDateTime>2010-10-21T01:02:09Z</StartDateTime>
+              <ReadyDateTime>2010-10-21T01:03:03Z</ReadyDateTime>
+              <State>ENDED</State>
+              <EndDateTime>2010-10-21T01:44:18Z</EndDateTime>
+              <InstanceRequestCount>1</InstanceRequestCount>
+              <InstanceType>m1.large</InstanceType>
+              <Market>ON_DEMAND</Market>
+              <LastStateChangeReason>Job flow terminated</LastStateChangeReason>
+              <InstanceRole>MASTER</InstanceRole>
+              <InstanceGroupId>ig-EVMHOZJ2SCO8</InstanceGroupId>
+              <Name>master</Name>
+            </member>
+            <member>
+              <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>
+              <InstanceRunningCount>0</InstanceRunningCount>
+              <StartDateTime>2010-10-21T01:03:59Z</StartDateTime>
+              <ReadyDateTime>2010-10-21T01:03:59Z</ReadyDateTime>
+              <State>ENDED</State>
+              <EndDateTime>2010-10-21T01:44:18Z</EndDateTime>
+              <InstanceRequestCount>9</InstanceRequestCount>
+              <InstanceType>m1.large</InstanceType>
+              <Market>ON_DEMAND</Market>
+              <LastStateChangeReason>Job flow terminated</LastStateChangeReason>
+              <InstanceRole>CORE</InstanceRole>
+              <InstanceGroupId>ig-YZHDYVITVHKB</InstanceGroupId>
+              <Name>slave</Name>
+            </member>
+          </InstanceGroups>
+          <NormalizedInstanceHours>40</NormalizedInstanceHours>
+          <HadoopVersion>0.20</HadoopVersion>
+          <MasterInstanceType>m1.large</MasterInstanceType>
+          <MasterPublicDnsName>ec2-184-72-153-139.compute-1.amazonaws.com</MasterPublicDnsName>
+          <Ec2KeyName>myubersecurekey</Ec2KeyName>
+          <InstanceCount>10</InstanceCount>
+          <KeepJobFlowAliveWhenNoSteps>false</KeepJobFlowAliveWhenNoSteps>
+        </Instances>
+      </member>
+    </JobFlows>
+  </DescribeJobFlowsResult>
+  <ResponseMetadata>
+    <RequestId>c31e701d-dcb4-11df-b5d9-337fc7fe4773</RequestId>
+  </ResponseMetadata>
+</DescribeJobFlowsResponse>
+"""
+
+
+class TestEMRResponses(unittest.TestCase):
+    def _parse_xml(self, body, markers):
+        rs = ResultSet(markers)
+        h = handler.XmlHandler(rs, None)
+        xml.sax.parseString(body, h)
+        return rs
+
+    def _assert_fields(self, response, **fields):
+        for field, expected in fields.items():
+            actual = getattr(response, field)
+            self.assertEquals(expected, actual,
+                              "Field %s: %r != %r" % (field, expected, actual))
+
+    def test_JobFlows_example(self):
+        [jobflow] = self._parse_xml(JOB_FLOW_EXAMPLE,
+                                    [('member', emrobject.JobFlow)])
+        self._assert_fields(jobflow,
+                            creationdatetime='2009-01-28T21:49:16Z',
+                            startdatetime='2009-01-28T21:49:16Z',
+                            state='STARTING',
+                            instancecount='4',
+                            jobflowid='j-3UN6WX5RRO2AG',
+                            loguri='mybucket/subdir/',
+                            name='MyJobFlowName',
+                            availabilityzone='us-east-1a',
+                            slaveinstancetype='m1.small',
+                            masterinstancetype='m1.small',
+                            ec2keyname='myec2keyname',
+                            keepjobflowalivewhennosteps='true')
+
+    def test_JobFlows_completed(self):
+        [jobflow] = self._parse_xml(JOB_FLOW_COMPLETED,
+                                    [('member', emrobject.JobFlow)])
+        self._assert_fields(jobflow,
+                            creationdatetime='2010-10-21T01:00:25Z',
+                            startdatetime='2010-10-21T01:03:59Z',
+                            enddatetime='2010-10-21T01:44:18Z',
+                            state='COMPLETED',
+                            instancecount='10',
+                            jobflowid='j-3H3Q13JPFLU22',
+                            loguri='s3n://example.emrtest.scripts/jobflow_logs/',
+                            name='RealJobFlowName',
+                            availabilityzone='us-east-1b',
+                            slaveinstancetype='m1.large',
+                            masterinstancetype='m1.large',
+                            ec2keyname='myubersecurekey',
+                            keepjobflowalivewhennosteps='false')
+        self.assertEquals(6, len(jobflow.steps))
+        self.assertEquals(2, len(jobflow.instancegroups))
+

data/packages/gsutil/boto/tests/unit/glacier/test_concurrent.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 Amazon.com, Inc. or its affiliates.  All Rights Reserved
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#
+from Queue import Queue
+
+import mock
+from tests.unit import unittest
+from tests.unit import AWSMockServiceTestCase
+
+from boto.glacier.concurrent import ConcurrentUploader, ConcurrentDownloader
+
+
+class FakeThreadedConcurrentUploader(ConcurrentUploader):
+    def _start_upload_threads(self, results_queue, upload_id,
+                              worker_queue, filename):
+        self.results_queue = results_queue
+        self.worker_queue = worker_queue
+        self.upload_id = upload_id
+
+    def _wait_for_upload_threads(self, hash_chunks, result_queue, total_parts):
+        for i in xrange(total_parts):
+            hash_chunks[i] = 'foo'
+
+class FakeThreadedConcurrentDownloader(ConcurrentDownloader):
+    def _start_download_threads(self, results_queue, worker_queue):
+        self.results_queue = results_queue
+        self.worker_queue = worker_queue
+
+    def _wait_for_download_threads(self, filename, result_queue, total_parts):
+        pass
+
+
+class TestConcurrentUploader(unittest.TestCase):
+
+    def setUp(self):
+        super(TestConcurrentUploader, self).setUp()
+        self.stat_patch = mock.patch('os.stat')
+        self.stat_mock = self.stat_patch.start()
+        # Give a default value for tests that don't care
+        # what the file size is.
+        self.stat_mock.return_value.st_size = 1024 * 1024 * 8
+
+    def tearDown(self):
+        self.stat_mock = self.stat_patch.start()
+
+    def test_calculate_required_part_size(self):
+        self.stat_mock.return_value.st_size = 1024 * 1024 * 8
+        uploader = ConcurrentUploader(mock.Mock(), 'vault_name')
+        total_parts, part_size = uploader._calculate_required_part_size(
+            1024 * 1024 * 8)
+        self.assertEqual(total_parts, 2)
+        self.assertEqual(part_size, 4 * 1024 * 1024)
+
+    def test_calculate_required_part_size_too_small(self):
+        too_small = 1 * 1024 * 1024
+        self.stat_mock.return_value.st_size = 1024 * 1024 * 1024
+        uploader = ConcurrentUploader(mock.Mock(), 'vault_name',
+                                      part_size=too_small)
+        total_parts, part_size = uploader._calculate_required_part_size(
+            1024 * 1024 * 1024)
+        self.assertEqual(total_parts, 256)
+        # Part size if 4MB not the passed in 1MB.
+        self.assertEqual(part_size, 4 * 1024 * 1024)
+
+    def test_work_queue_is_correctly_populated(self):
+        uploader = FakeThreadedConcurrentUploader(mock.MagicMock(),
+                                                  'vault_name')
+        uploader.upload('foofile')
+        q = uploader.worker_queue
+        items = [q.get() for i in xrange(q.qsize())]
+        self.assertEqual(items[0], (0, 4 * 1024 * 1024))
+        self.assertEqual(items[1], (1, 4 * 1024 * 1024))
+        # 2 for the parts, 10 for the end sentinels (10 threads).
+        self.assertEqual(len(items), 12)
+
+    def test_correct_low_level_api_calls(self):
+        api_mock = mock.MagicMock()
+        uploader = FakeThreadedConcurrentUploader(api_mock, 'vault_name')
+        uploader.upload('foofile')
+        # The threads call the upload_part, so we're just verifying the
+        # initiate/complete multipart API calls.
+        api_mock.initiate_multipart_upload.assert_called_with(
+            'vault_name', 4 * 1024 * 1024, None)
+        api_mock.complete_multipart_upload.assert_called_with(
+            'vault_name', mock.ANY, mock.ANY, 8 * 1024 * 1024)
+
+    def test_downloader_work_queue_is_correctly_populated(self):
+        job = mock.MagicMock()
+        job.archive_size = 8 * 1024 * 1024
+        downloader = FakeThreadedConcurrentDownloader(job)
+        downloader.download('foofile')
+        q = downloader.worker_queue
+        items = [q.get() for i in xrange(q.qsize())]
+        self.assertEqual(items[0], (0, 4 * 1024 * 1024))
+        self.assertEqual(items[1], (1, 4 * 1024 * 1024))
+        # 2 for the parts, 10 for the end sentinels (10 threads).
+        self.assertEqual(len(items), 12)
+
+
+if __name__ == '__main__':
+    unittest.main()