libv8 4.5.95.5 → 5.0.71.48.0beta2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +45 -19
- data/CHANGELOG.md +14 -0
- data/README.md +30 -15
- data/Rakefile +7 -6
- data/ext/libv8/arch.rb +5 -4
- data/ext/libv8/builder.rb +25 -19
- data/ext/libv8/compiler.rb +6 -33
- data/ext/libv8/location.rb +7 -8
- data/lib/libv8/version.rb +1 -1
- data/libv8.gemspec +1 -1
- data/patches/build-standalone-static-library.patch +14 -0
- data/patches/disable-building-tests.patch +48 -10
- data/patches/fPIC-for-static.patch +3 -3
- data/release/x86-linux/Vagrantfile +8 -4
- data/release/x86_64-freebsd10/Vagrantfile +86 -0
- data/release/x86_64-linux/Vagrantfile +8 -4
- data/spec/compiler_spec.rb +5 -29
- data/spec/support/compiler_helpers.rb +2 -4
- data/vendor/depot_tools/.gitignore +15 -3
- data/vendor/depot_tools/OWNERS +2 -2
- data/vendor/depot_tools/PRESUBMIT.py +4 -2
- data/vendor/depot_tools/WATCHLISTS +6 -0
- data/vendor/depot_tools/apply_issue.py +70 -38
- data/vendor/depot_tools/bootstrap/win/README.md +66 -0
- data/vendor/depot_tools/bootstrap/win/git-bash.template.sh +12 -0
- data/vendor/depot_tools/bootstrap/win/git.template.bat +5 -0
- data/vendor/depot_tools/bootstrap/win/profile.d.python.sh +20 -0
- data/vendor/depot_tools/bootstrap/win/win_tools.bat +96 -45
- data/vendor/depot_tools/breakpad.py +6 -141
- data/vendor/depot_tools/buildbucket.py +45 -31
- data/vendor/depot_tools/cbuildbot +1 -0
- data/vendor/depot_tools/checkout.py +2 -1
- data/vendor/depot_tools/chrome_set_ver +1 -0
- data/vendor/depot_tools/cit +8 -0
- data/vendor/depot_tools/cit.bat +11 -0
- data/vendor/depot_tools/cit.py +120 -0
- data/vendor/depot_tools/codereview.settings +0 -2
- data/vendor/depot_tools/commit_queue +1 -5
- data/vendor/depot_tools/commit_queue.bat +1 -4
- data/vendor/depot_tools/commit_queue.py +78 -29
- data/vendor/depot_tools/cpplint.py +22 -14
- data/vendor/depot_tools/cros +1 -0
- data/vendor/depot_tools/cros_sdk +1 -0
- data/vendor/depot_tools/depot-tools-auth.py +3 -3
- data/vendor/depot_tools/download_from_google_storage.py +101 -21
- data/vendor/depot_tools/drover.py +2 -3
- data/vendor/depot_tools/fetch.py +31 -27
- data/vendor/depot_tools/{recipes → fetch_configs}/android.py +4 -4
- data/vendor/depot_tools/fetch_configs/breakpad.py +45 -0
- data/vendor/depot_tools/{recipes → fetch_configs}/chromium.py +3 -3
- data/vendor/depot_tools/{recipes/recipe_util.py → fetch_configs/config_util.py} +3 -3
- data/vendor/depot_tools/fetch_configs/crashpad.py +41 -0
- data/vendor/depot_tools/{recipes → fetch_configs}/dart.py +3 -3
- data/vendor/depot_tools/{recipes/pdfium.py → fetch_configs/dartino.py} +14 -13
- data/vendor/depot_tools/{recipes → fetch_configs}/dartium.py +3 -3
- data/vendor/depot_tools/{recipes → fetch_configs}/depot_tools.py +3 -3
- data/vendor/depot_tools/fetch_configs/gyp.py +42 -0
- data/vendor/depot_tools/{recipes → fetch_configs}/infra.py +3 -3
- data/vendor/depot_tools/{recipes → fetch_configs}/infra_internal.py +3 -3
- data/vendor/depot_tools/{recipes → fetch_configs}/ios.py +4 -4
- data/vendor/depot_tools/{recipes → fetch_configs}/mojo.py +3 -3
- data/vendor/depot_tools/{recipes → fetch_configs}/nacl.py +3 -3
- data/vendor/depot_tools/{recipes → fetch_configs}/naclports.py +3 -3
- data/vendor/depot_tools/fetch_configs/pdfium.py +40 -0
- data/vendor/depot_tools/{recipes → fetch_configs}/skia.py +3 -3
- data/vendor/depot_tools/{recipes → fetch_configs}/skia_buildbot.py +3 -3
- data/vendor/depot_tools/fetch_configs/syzygy.py +41 -0
- data/vendor/depot_tools/{recipes → fetch_configs}/v8.py +3 -3
- data/vendor/depot_tools/{recipes → fetch_configs}/webrtc.py +3 -3
- data/vendor/depot_tools/{recipes → fetch_configs}/webrtc_android.py +4 -4
- data/vendor/depot_tools/{recipes → fetch_configs}/webrtc_ios.py +4 -4
- data/vendor/depot_tools/fix_encoding.py +6 -6
- data/vendor/depot_tools/gcl.py +11 -21
- data/vendor/depot_tools/gclient +10 -0
- data/vendor/depot_tools/gclient-new-workdir.py +7 -38
- data/vendor/depot_tools/gclient.bat +2 -2
- data/vendor/depot_tools/gclient.py +85 -65
- data/vendor/depot_tools/gclient_scm.py +83 -10
- data/vendor/depot_tools/gclient_utils.py +5 -1
- data/vendor/depot_tools/gerrit_util.py +243 -26
- data/vendor/depot_tools/git-auto-svn +1 -1
- data/vendor/depot_tools/git-cache +1 -1
- data/vendor/depot_tools/git-cherry-pick-upload +1 -1
- data/vendor/depot_tools/git-cl +1 -1
- data/vendor/depot_tools/git-drover +6 -0
- data/vendor/depot_tools/git-find-releases +6 -0
- data/vendor/depot_tools/git-footers +1 -1
- data/vendor/depot_tools/git-freeze +1 -1
- data/vendor/depot_tools/git-gs +1 -1
- data/vendor/depot_tools/git-hyper-blame +6 -0
- data/vendor/depot_tools/git-map +1 -1
- data/vendor/depot_tools/git-map-branches +1 -1
- data/vendor/depot_tools/git-mark-merge-base +1 -1
- data/vendor/depot_tools/git-nav-downstream +1 -1
- data/vendor/depot_tools/git-new-branch +1 -1
- data/vendor/depot_tools/git-number +1 -1
- data/vendor/depot_tools/git-rebase-update +1 -1
- data/vendor/depot_tools/git-rename-branch +1 -1
- data/vendor/depot_tools/git-reparent-branch +1 -1
- data/vendor/depot_tools/git-retry +1 -1
- data/vendor/depot_tools/git-squash-branch +1 -1
- data/vendor/depot_tools/git-thaw +1 -1
- data/vendor/depot_tools/git-try +1 -1
- data/vendor/depot_tools/git-upstream-diff +1 -1
- data/vendor/depot_tools/git_auto_svn.py +24 -6
- data/vendor/depot_tools/git_cache.py +74 -27
- data/vendor/depot_tools/git_cl.py +2118 -747
- data/vendor/depot_tools/git_common.py +100 -6
- data/vendor/depot_tools/git_dates.py +62 -0
- data/vendor/depot_tools/git_drover.py +424 -0
- data/vendor/depot_tools/git_find_releases.py +65 -0
- data/vendor/depot_tools/git_footers.py +42 -0
- data/vendor/depot_tools/git_hyper_blame.py +391 -0
- data/vendor/depot_tools/git_map_branches.py +8 -6
- data/vendor/depot_tools/git_new_branch.py +6 -1
- data/vendor/depot_tools/git_rebase_update.py +56 -16
- data/vendor/depot_tools/git_reparent_branch.py +13 -0
- data/vendor/depot_tools/git_try.py +0 -2
- data/vendor/depot_tools/gsutil.py +51 -20
- data/vendor/depot_tools/infra/config/OWNERS +3 -1
- data/vendor/depot_tools/infra/config/cq.cfg +7 -3
- data/vendor/depot_tools/infra/config/recipes.cfg +9 -0
- data/vendor/depot_tools/luci_hacks/README.md +35 -0
- data/vendor/depot_tools/{bootstrap/virtualenv/tests → luci_hacks}/__init__.py +0 -0
- data/vendor/depot_tools/luci_hacks/luci_recipe_run.isolate +12 -0
- data/vendor/depot_tools/luci_hacks/luci_recipe_run.py +81 -0
- data/vendor/depot_tools/luci_hacks/trigger_luci_job.py +128 -0
- data/vendor/depot_tools/man/html/depot_tools.html +9 -1
- data/vendor/depot_tools/man/html/depot_tools_tutorial.html +4 -4
- data/vendor/depot_tools/man/html/git-drover.html +191 -35
- data/vendor/depot_tools/man/html/git-hyper-blame.html +878 -0
- data/vendor/depot_tools/man/html/git-rebase-update.html +9 -4
- data/vendor/depot_tools/man/man1/git-drover.1 +189 -36
- data/vendor/depot_tools/man/man1/git-hyper-blame.1 +128 -0
- data/vendor/depot_tools/man/man1/git-rebase-update.1 +8 -6
- data/vendor/depot_tools/man/man7/depot_tools.7 +9 -4
- data/vendor/depot_tools/man/src/_git-hyper-blame_desc.helper.txt +1 -0
- data/vendor/depot_tools/man/src/common_demo_functions.sh +5 -0
- data/vendor/depot_tools/man/src/depot_tools_tutorial.txt +1 -1
- data/vendor/depot_tools/man/src/git-drover.demo.1.sh +11 -16
- data/vendor/depot_tools/man/src/git-drover.demo.3.sh +27 -0
- data/vendor/depot_tools/man/src/git-drover.demo.4.sh +39 -0
- data/vendor/depot_tools/man/src/git-drover.txt +49 -3
- data/vendor/depot_tools/man/src/git-hyper-blame.demo.1.sh +3 -0
- data/vendor/depot_tools/man/src/git-hyper-blame.demo.2.sh +4 -0
- data/vendor/depot_tools/man/src/git-hyper-blame.demo.common.sh +57 -0
- data/vendor/depot_tools/man/src/git-hyper-blame.txt +85 -0
- data/vendor/depot_tools/man/src/git-rebase-update.txt +5 -1
- data/vendor/depot_tools/my_activity.py +6 -21
- data/vendor/depot_tools/ninja +2 -2
- data/vendor/depot_tools/ninja-linux32 +0 -0
- data/vendor/depot_tools/ninja-linux64 +0 -0
- data/vendor/depot_tools/ninja-mac +0 -0
- data/vendor/depot_tools/ninja.exe +0 -0
- data/vendor/depot_tools/presubmit_canned_checks.py +83 -69
- data/vendor/depot_tools/presubmit_support.py +126 -42
- data/vendor/depot_tools/pylint.py +5 -1
- data/vendor/depot_tools/python_runner.sh +55 -0
- data/vendor/depot_tools/recipe_modules/bot_update/__init__.py +32 -0
- data/vendor/depot_tools/recipe_modules/bot_update/api.py +283 -0
- data/vendor/depot_tools/recipe_modules/bot_update/example.expected/basic.json +56 -0
- data/vendor/depot_tools/recipe_modules/bot_update/example.expected/basic_output_manifest.json +63 -0
- data/vendor/depot_tools/recipe_modules/bot_update/example.expected/basic_with_branch_heads.json +57 -0
- data/vendor/depot_tools/recipe_modules/bot_update/example.expected/clobber.json +44 -0
- data/vendor/depot_tools/recipe_modules/bot_update/example.expected/forced.json +57 -0
- data/vendor/depot_tools/recipe_modules/bot_update/example.expected/gerrit_no_reset.json +44 -0
- data/vendor/depot_tools/recipe_modules/bot_update/example.expected/no_shallow.json +44 -0
- data/vendor/depot_tools/recipe_modules/bot_update/example.expected/off.json +43 -0
- data/vendor/depot_tools/recipe_modules/bot_update/example.expected/reset_root_solution_revision.json +43 -0
- data/vendor/depot_tools/recipe_modules/bot_update/example.expected/svn_mode.json +59 -0
- data/vendor/depot_tools/recipe_modules/bot_update/example.expected/trychange.json +58 -0
- data/vendor/depot_tools/recipe_modules/bot_update/example.expected/trychange_oauth2.json +60 -0
- data/vendor/depot_tools/recipe_modules/bot_update/example.expected/tryjob.json +58 -0
- data/vendor/depot_tools/recipe_modules/bot_update/example.expected/tryjob_fail.json +60 -0
- data/vendor/depot_tools/recipe_modules/bot_update/example.expected/tryjob_fail_patch.json +81 -0
- data/vendor/depot_tools/recipe_modules/bot_update/example.expected/tryjob_fail_patch_download.json +81 -0
- data/vendor/depot_tools/recipe_modules/bot_update/example.expected/tryjob_gerrit_angle.json +49 -0
- data/vendor/depot_tools/recipe_modules/bot_update/example.expected/tryjob_v8.json +61 -0
- data/vendor/depot_tools/recipe_modules/bot_update/example.expected/tryjob_v8_head_by_default.json +51 -0
- data/vendor/depot_tools/recipe_modules/bot_update/example.py +172 -0
- data/vendor/depot_tools/{bootstrap/virtualenv/virtualenv_support → recipe_modules/bot_update/resources}/__init__.py +0 -0
- data/vendor/depot_tools/recipe_modules/bot_update/resources/bot_update.py +1764 -0
- data/vendor/depot_tools/recipe_modules/bot_update/test_api.py +86 -0
- data/vendor/depot_tools/recipe_modules/depot_tools/__init__.py +3 -0
- data/vendor/depot_tools/recipe_modules/depot_tools/api.py +27 -0
- data/vendor/depot_tools/recipe_modules/gclient/__init__.py +10 -0
- data/vendor/depot_tools/recipe_modules/gclient/api.py +378 -0
- data/vendor/depot_tools/recipe_modules/gclient/config.py +671 -0
- data/vendor/depot_tools/recipe_modules/gclient/example.expected/basic.json +172 -0
- data/vendor/depot_tools/recipe_modules/gclient/example.expected/revision.json +174 -0
- data/vendor/depot_tools/recipe_modules/gclient/example.expected/tryserver.json +185 -0
- data/vendor/depot_tools/recipe_modules/gclient/example.py +100 -0
- data/vendor/depot_tools/recipe_modules/gclient/test_api.py +37 -0
- data/vendor/depot_tools/recipe_modules/git/__init__.py +9 -0
- data/vendor/depot_tools/recipe_modules/git/api.py +377 -0
- data/vendor/depot_tools/recipe_modules/git/example.expected/basic.json +177 -0
- data/vendor/depot_tools/recipe_modules/git/example.expected/basic_branch.json +177 -0
- data/vendor/depot_tools/recipe_modules/git/example.expected/basic_file_name.json +179 -0
- data/vendor/depot_tools/recipe_modules/git/example.expected/basic_hash.json +176 -0
- data/vendor/depot_tools/recipe_modules/git/example.expected/basic_ref.json +177 -0
- data/vendor/depot_tools/recipe_modules/git/example.expected/basic_submodule_update_force.json +178 -0
- data/vendor/depot_tools/recipe_modules/git/example.expected/can_fail_build.json +153 -0
- data/vendor/depot_tools/recipe_modules/git/example.expected/cannot_fail_build.json +181 -0
- data/vendor/depot_tools/recipe_modules/git/example.expected/cat-file_test.json +199 -0
- data/vendor/depot_tools/recipe_modules/git/example.expected/count-objects_delta.json +250 -0
- data/vendor/depot_tools/recipe_modules/git/example.expected/count-objects_failed.json +181 -0
- data/vendor/depot_tools/recipe_modules/git/example.expected/count-objects_with_bad_output.json +182 -0
- data/vendor/depot_tools/recipe_modules/git/example.expected/count-objects_with_bad_output_fails_build.json +102 -0
- data/vendor/depot_tools/recipe_modules/git/example.expected/curl_trace_file.json +181 -0
- data/vendor/depot_tools/recipe_modules/git/example.expected/platform_win.json +186 -0
- data/vendor/depot_tools/recipe_modules/git/example.expected/rebase_failed.json +179 -0
- data/vendor/depot_tools/recipe_modules/git/example.expected/remote_not_origin.json +179 -0
- data/vendor/depot_tools/recipe_modules/git/example.expected/set_got_revision.json +178 -0
- data/vendor/depot_tools/recipe_modules/git/example.py +147 -0
- data/vendor/depot_tools/recipe_modules/git/resources/git_setup.py +61 -0
- data/vendor/depot_tools/recipe_modules/git/test_api.py +18 -0
- data/vendor/depot_tools/recipe_modules/git_cl/__init__.py +4 -0
- data/vendor/depot_tools/recipe_modules/git_cl/api.py +25 -0
- data/vendor/depot_tools/recipe_modules/git_cl/config.py +22 -0
- data/vendor/depot_tools/recipe_modules/git_cl/example.expected/basic.json +66 -0
- data/vendor/depot_tools/recipe_modules/git_cl/example.py +41 -0
- data/vendor/depot_tools/recipe_modules/infra_paths/__init__.py +4 -0
- data/vendor/depot_tools/recipe_modules/infra_paths/api.py +12 -0
- data/vendor/depot_tools/recipe_modules/infra_paths/example.expected/basic.json +14 -0
- data/vendor/depot_tools/recipe_modules/infra_paths/example.expected/paths_buildbot_linux.json +14 -0
- data/vendor/depot_tools/recipe_modules/infra_paths/example.expected/paths_buildbot_mac.json +14 -0
- data/vendor/depot_tools/recipe_modules/infra_paths/example.expected/paths_buildbot_win.json +14 -0
- data/vendor/depot_tools/recipe_modules/infra_paths/example.expected/paths_kitchen_linux.json +14 -0
- data/vendor/depot_tools/recipe_modules/infra_paths/example.expected/paths_kitchen_mac.json +14 -0
- data/vendor/depot_tools/recipe_modules/infra_paths/example.expected/paths_kitchen_win.json +14 -0
- data/vendor/depot_tools/recipe_modules/infra_paths/example.py +28 -0
- data/vendor/depot_tools/recipe_modules/infra_paths/path_config.py +45 -0
- data/vendor/depot_tools/recipe_modules/presubmit/__init__.py +4 -0
- data/vendor/depot_tools/recipe_modules/presubmit/api.py +20 -0
- data/vendor/depot_tools/recipe_modules/presubmit/example.expected/basic.json +18 -0
- data/vendor/depot_tools/recipe_modules/presubmit/example.py +15 -0
- data/vendor/depot_tools/recipe_modules/rietveld/__init__.py +5 -0
- data/vendor/depot_tools/recipe_modules/rietveld/api.py +94 -0
- data/vendor/depot_tools/recipe_modules/rietveld/example.expected/basic.json +30 -0
- data/vendor/depot_tools/recipe_modules/rietveld/example.py +24 -0
- data/vendor/depot_tools/recipe_modules/tryserver/__init__.py +15 -0
- data/vendor/depot_tools/recipe_modules/tryserver/api.py +280 -0
- data/vendor/depot_tools/recipe_modules/tryserver/example.expected/with_git_patch.json +104 -0
- data/vendor/depot_tools/recipe_modules/tryserver/example.expected/with_rietveld_patch.json +58 -0
- data/vendor/depot_tools/recipe_modules/tryserver/example.expected/with_rietveld_patch_new.json +58 -0
- data/vendor/depot_tools/recipe_modules/tryserver/example.expected/with_svn_patch.json +68 -0
- data/vendor/depot_tools/recipe_modules/tryserver/example.expected/with_wrong_patch.json +43 -0
- data/vendor/depot_tools/recipe_modules/tryserver/example.expected/with_wrong_patch_new.json +43 -0
- data/vendor/depot_tools/recipe_modules/tryserver/example.py +53 -0
- data/vendor/depot_tools/recipe_modules/tryserver/test_api.py +7 -0
- data/vendor/depot_tools/recipes.py +136 -0
- data/vendor/depot_tools/repo +1 -1
- data/vendor/depot_tools/rietveld.py +46 -15
- data/vendor/depot_tools/roll_dep.py +97 -36
- data/vendor/depot_tools/scm.py +3 -3
- data/vendor/depot_tools/setup_color.py +94 -0
- data/vendor/depot_tools/subprocess2.py +10 -1
- data/vendor/depot_tools/third_party/cq_client/OWNERS +0 -1
- data/vendor/depot_tools/third_party/cq_client/README.md +47 -9
- data/vendor/depot_tools/third_party/cq_client/cq.pb.go +617 -0
- data/vendor/depot_tools/third_party/cq_client/cq.proto +75 -17
- data/vendor/depot_tools/third_party/cq_client/cq_pb2.py +168 -41
- data/vendor/depot_tools/third_party/cq_client/testdata/cq_gerrit.cfg +55 -0
- data/vendor/depot_tools/third_party/cq_client/{test/cq_example.cfg → testdata/cq_rietveld.cfg} +14 -6
- data/vendor/depot_tools/third_party/fancy_urllib/README +5 -4
- data/vendor/depot_tools/third_party/fancy_urllib/__init__.py +114 -52
- data/vendor/depot_tools/third_party/protobuf26/README.chromium +9 -6
- data/vendor/depot_tools/third_party/upload.py +17 -31
- data/vendor/depot_tools/trychange.py +0 -2
- data/vendor/depot_tools/update_depot_tools +29 -11
- data/vendor/depot_tools/update_depot_tools.bat +4 -9
- data/vendor/depot_tools/upload_to_google_storage.py +42 -5
- data/vendor/depot_tools/win_toolchain/OWNERS +1 -0
- data/vendor/depot_tools/win_toolchain/get_toolchain_if_necessary.py +227 -52
- data/vendor/depot_tools/win_toolchain/package_from_installed.py +203 -88
- metadata +161 -81
- data/patches/arm/do-not-imply-vfp3-and-armv7.patch +0 -16
- data/patches/arm/do-not-use-vfp2.patch +0 -13
- data/patches/clang51/no-unused-variable.patch +0 -12
- data/vendor/depot_tools/bootstrap/.gitignore +0 -2
- data/vendor/depot_tools/bootstrap/bootstrap.py +0 -234
- data/vendor/depot_tools/bootstrap/deps.pyl +0 -15
- data/vendor/depot_tools/bootstrap/util.py +0 -87
- data/vendor/depot_tools/bootstrap/virtualenv/.gitignore +0 -10
- data/vendor/depot_tools/bootstrap/virtualenv/.travis.yml +0 -28
- data/vendor/depot_tools/bootstrap/virtualenv/AUTHORS.txt +0 -91
- data/vendor/depot_tools/bootstrap/virtualenv/CONTRIBUTING.rst +0 -21
- data/vendor/depot_tools/bootstrap/virtualenv/LICENSE.txt +0 -22
- data/vendor/depot_tools/bootstrap/virtualenv/MANIFEST.in +0 -11
- data/vendor/depot_tools/bootstrap/virtualenv/README.rst +0 -10
- data/vendor/depot_tools/bootstrap/virtualenv/bin/rebuild-script.py +0 -71
- data/vendor/depot_tools/bootstrap/virtualenv/docs/changes.rst +0 -747
- data/vendor/depot_tools/bootstrap/virtualenv/docs/conf.py +0 -149
- data/vendor/depot_tools/bootstrap/virtualenv/docs/development.rst +0 -61
- data/vendor/depot_tools/bootstrap/virtualenv/docs/index.rst +0 -137
- data/vendor/depot_tools/bootstrap/virtualenv/docs/installation.rst +0 -58
- data/vendor/depot_tools/bootstrap/virtualenv/docs/make.bat +0 -170
- data/vendor/depot_tools/bootstrap/virtualenv/docs/reference.rst +0 -256
- data/vendor/depot_tools/bootstrap/virtualenv/docs/userguide.rst +0 -249
- data/vendor/depot_tools/bootstrap/virtualenv/scripts/virtualenv +0 -3
- data/vendor/depot_tools/bootstrap/virtualenv/setup.py +0 -111
- data/vendor/depot_tools/bootstrap/virtualenv/tests/test_activate.sh +0 -94
- data/vendor/depot_tools/bootstrap/virtualenv/tests/test_activate_expected.output +0 -2
- data/vendor/depot_tools/bootstrap/virtualenv/tests/test_virtualenv.py +0 -139
- data/vendor/depot_tools/bootstrap/virtualenv/tests/tox.ini +0 -12
- data/vendor/depot_tools/bootstrap/virtualenv/tox.ini +0 -17
- data/vendor/depot_tools/bootstrap/virtualenv/virtualenv.py +0 -2367
- data/vendor/depot_tools/bootstrap/virtualenv/virtualenv_embedded/activate.bat +0 -26
- data/vendor/depot_tools/bootstrap/virtualenv/virtualenv_embedded/activate.csh +0 -42
- data/vendor/depot_tools/bootstrap/virtualenv/virtualenv_embedded/activate.fish +0 -74
- data/vendor/depot_tools/bootstrap/virtualenv/virtualenv_embedded/activate.ps1 +0 -150
- data/vendor/depot_tools/bootstrap/virtualenv/virtualenv_embedded/activate.sh +0 -80
- data/vendor/depot_tools/bootstrap/virtualenv/virtualenv_embedded/activate_this.py +0 -34
- data/vendor/depot_tools/bootstrap/virtualenv/virtualenv_embedded/deactivate.bat +0 -20
- data/vendor/depot_tools/bootstrap/virtualenv/virtualenv_embedded/distutils-init.py +0 -101
- data/vendor/depot_tools/bootstrap/virtualenv/virtualenv_embedded/distutils.cfg +0 -6
- data/vendor/depot_tools/bootstrap/virtualenv/virtualenv_embedded/site.py +0 -758
- data/vendor/depot_tools/bootstrap/virtualenv/virtualenv_support/pip-6.0-py2.py3-none-any.whl +0 -0
- data/vendor/depot_tools/bootstrap/virtualenv/virtualenv_support/setuptools-8.2.1-py2.py3-none-any.whl +0 -0
- data/vendor/depot_tools/bootstrap/win/README.google +0 -16
- data/vendor/depot_tools/cbuildbot +0 -96
- data/vendor/depot_tools/chrome_set_ver +0 -96
- data/vendor/depot_tools/cros +0 -96
- data/vendor/depot_tools/cros_sdk +0 -96
- data/vendor/depot_tools/git-cl-upload-hook +0 -52
- data/vendor/depot_tools/git-crup +0 -45
- data/vendor/depot_tools/python_git_runner.sh +0 -36
- data/vendor/depot_tools/recipes/blink.py +0 -59
- data/vendor/depot_tools/third_party/cq_client/test/validate_config_test.py +0 -52
- data/vendor/depot_tools/third_party/cq_client/validate_config.py +0 -108
- data/vendor/depot_tools/win_toolchain/toolchain2013.py +0 -494
data/vendor/depot_tools/recipe_modules/bot_update/example.expected/tryjob_v8_head_by_default.json
ADDED
@@ -0,0 +1,51 @@
|
|
1
|
+
[
|
2
|
+
{
|
3
|
+
"cmd": [
|
4
|
+
"python",
|
5
|
+
"-u",
|
6
|
+
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
|
7
|
+
"--master",
|
8
|
+
"chromium.testing.master",
|
9
|
+
"--builder",
|
10
|
+
"TestBuilder",
|
11
|
+
"--slave",
|
12
|
+
"TestSlavename",
|
13
|
+
"--spec",
|
14
|
+
"cache_dir = '[GIT_CACHE]'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]",
|
15
|
+
"--root",
|
16
|
+
"src/v8",
|
17
|
+
"--revision_mapping_file",
|
18
|
+
"{\"src\": \"got_cr_revision\"}",
|
19
|
+
"--git-cache-dir",
|
20
|
+
"[GIT_CACHE]",
|
21
|
+
"--issue",
|
22
|
+
"12853011",
|
23
|
+
"--patchset",
|
24
|
+
"1",
|
25
|
+
"--rietveld_server",
|
26
|
+
"https://codereview.chromium.org",
|
27
|
+
"--output_json",
|
28
|
+
"/path/to/tmp/json",
|
29
|
+
"--revision",
|
30
|
+
"src@HEAD",
|
31
|
+
"--revision",
|
32
|
+
"src/v8@HEAD"
|
33
|
+
],
|
34
|
+
"env": {
|
35
|
+
"PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]"
|
36
|
+
},
|
37
|
+
"name": "bot_update",
|
38
|
+
"~followup_annotations": [
|
39
|
+
"@@@STEP_LOG_LINE@json.output@{@@@",
|
40
|
+
"@@@STEP_LOG_LINE@json.output@ \"did_run\": false, @@@",
|
41
|
+
"@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false@@@",
|
42
|
+
"@@@STEP_LOG_LINE@json.output@}@@@",
|
43
|
+
"@@@STEP_LOG_END@json.output@@@"
|
44
|
+
]
|
45
|
+
},
|
46
|
+
{
|
47
|
+
"name": "$result",
|
48
|
+
"recipe_result": null,
|
49
|
+
"status_code": 0
|
50
|
+
}
|
51
|
+
]
|
@@ -0,0 +1,172 @@
|
|
1
|
+
# Copyright 2014 The Chromium Authors. All rights reserved.
|
2
|
+
# Use of this source code is governed by a BSD-style license that can be
|
3
|
+
# found in the LICENSE file.
|
4
|
+
|
5
|
+
DEPS = [
|
6
|
+
'bot_update',
|
7
|
+
'gclient',
|
8
|
+
'recipe_engine/path',
|
9
|
+
'recipe_engine/properties',
|
10
|
+
]
|
11
|
+
|
12
|
+
def RunSteps(api):
|
13
|
+
api.gclient.use_mirror = True
|
14
|
+
|
15
|
+
src_cfg = api.gclient.make_config(GIT_MODE=True, CACHE_DIR='[GIT_CACHE]')
|
16
|
+
soln = src_cfg.solutions.add()
|
17
|
+
soln.name = 'src'
|
18
|
+
soln.url = 'svn://svn.chromium.org/chrome/trunk/src'
|
19
|
+
soln.revision = api.properties.get('revision')
|
20
|
+
api.gclient.c = src_cfg
|
21
|
+
api.gclient.c.revisions.update(api.properties.get('revisions', {}))
|
22
|
+
api.gclient.c.got_revision_mapping['src'] = 'got_cr_revision'
|
23
|
+
api.gclient.c.patch_projects['v8'] = ('src/v8', 'HEAD')
|
24
|
+
api.gclient.c.patch_projects['angle/angle'] = ('src/third_party/angle',
|
25
|
+
'HEAD')
|
26
|
+
patch = api.properties.get('patch', True)
|
27
|
+
clobber = True if api.properties.get('clobber') else False
|
28
|
+
force = True if api.properties.get('force') else False
|
29
|
+
no_shallow = True if api.properties.get('no_shallow') else False
|
30
|
+
output_manifest = api.properties.get('output_manifest', False)
|
31
|
+
with_branch_heads = api.properties.get('with_branch_heads', False)
|
32
|
+
refs = api.properties.get('refs', [])
|
33
|
+
oauth2 = api.properties.get('oauth2', False)
|
34
|
+
root_solution_revision = api.properties.get('root_solution_revision')
|
35
|
+
suffix = api.properties.get('suffix')
|
36
|
+
gerrit_no_reset = True if api.properties.get('gerrit_no_reset') else False
|
37
|
+
api.bot_update.ensure_checkout(force=force,
|
38
|
+
no_shallow=no_shallow,
|
39
|
+
patch=patch,
|
40
|
+
with_branch_heads=with_branch_heads,
|
41
|
+
output_manifest=output_manifest,
|
42
|
+
refs=refs, patch_oauth2=oauth2,
|
43
|
+
clobber=clobber,
|
44
|
+
root_solution_revision=root_solution_revision,
|
45
|
+
suffix=suffix,
|
46
|
+
gerrit_no_reset=gerrit_no_reset)
|
47
|
+
|
48
|
+
|
49
|
+
def GenTests(api):
|
50
|
+
yield api.test('basic') + api.properties(
|
51
|
+
mastername='chromium.linux',
|
52
|
+
buildername='Linux Builder',
|
53
|
+
slavename='totallyaslave-m1',
|
54
|
+
patch=False,
|
55
|
+
revision='abc'
|
56
|
+
)
|
57
|
+
yield api.test('basic_with_branch_heads') + api.properties(
|
58
|
+
mastername='chromium.linux',
|
59
|
+
buildername='Linux Builder',
|
60
|
+
slavename='totallyaslave-m1',
|
61
|
+
with_branch_heads=True,
|
62
|
+
suffix='with branch heads'
|
63
|
+
)
|
64
|
+
yield api.test('basic_output_manifest') + api.properties(
|
65
|
+
mastername='chromium.linux',
|
66
|
+
buildername='Linux Builder',
|
67
|
+
slavename='totallyaslave-m1',
|
68
|
+
output_manifest=True,
|
69
|
+
)
|
70
|
+
yield api.test('tryjob') + api.properties(
|
71
|
+
mastername='tryserver.chromium.linux',
|
72
|
+
buildername='linux_rel',
|
73
|
+
slavename='totallyaslave-c4',
|
74
|
+
issue=12345,
|
75
|
+
patchset=654321,
|
76
|
+
patch_url='http://src.chromium.org/foo/bar'
|
77
|
+
)
|
78
|
+
yield api.test('trychange') + api.properties(
|
79
|
+
mastername='tryserver.chromium.linux',
|
80
|
+
buildername='linux_rel',
|
81
|
+
slavename='totallyaslave-c4',
|
82
|
+
refs=['+refs/change/1/2/333'],
|
83
|
+
)
|
84
|
+
yield api.test('trychange_oauth2') + api.properties(
|
85
|
+
mastername='tryserver.chromium.linux',
|
86
|
+
buildername='linux_rel',
|
87
|
+
slavename='totallyaslave-c4',
|
88
|
+
oauth2=True,
|
89
|
+
)
|
90
|
+
yield api.test('tryjob_fail') + api.properties(
|
91
|
+
mastername='tryserver.chromium.linux',
|
92
|
+
buildername='linux_rel',
|
93
|
+
slavename='totallyaslave-c4',
|
94
|
+
issue=12345,
|
95
|
+
patchset=654321,
|
96
|
+
patch_url='http://src.chromium.org/foo/bar',
|
97
|
+
) + api.step_data('bot_update', retcode=1)
|
98
|
+
yield api.test('tryjob_fail_patch') + api.properties(
|
99
|
+
mastername='tryserver.chromium.linux',
|
100
|
+
buildername='linux_rel',
|
101
|
+
slavename='totallyaslave-c4',
|
102
|
+
issue=12345,
|
103
|
+
patchset=654321,
|
104
|
+
patch_url='http://src.chromium.org/foo/bar',
|
105
|
+
fail_patch='apply',
|
106
|
+
) + api.step_data('bot_update', retcode=88)
|
107
|
+
yield api.test('tryjob_fail_patch_download') + api.properties(
|
108
|
+
mastername='tryserver.chromium.linux',
|
109
|
+
buildername='linux_rel',
|
110
|
+
slavename='totallyaslave-c4',
|
111
|
+
issue=12345,
|
112
|
+
patchset=654321,
|
113
|
+
patch_url='http://src.chromium.org/foo/bar',
|
114
|
+
fail_patch='download'
|
115
|
+
) + api.step_data('bot_update', retcode=87)
|
116
|
+
yield api.test('forced') + api.properties(
|
117
|
+
mastername='experimental',
|
118
|
+
buildername='Experimental Builder',
|
119
|
+
slavename='somehost',
|
120
|
+
force=1
|
121
|
+
)
|
122
|
+
yield api.test('no_shallow') + api.properties(
|
123
|
+
mastername='experimental',
|
124
|
+
buildername='Experimental Builder',
|
125
|
+
slavename='somehost',
|
126
|
+
no_shallow=1
|
127
|
+
)
|
128
|
+
yield api.test('off') + api.properties(
|
129
|
+
mastername='experimental',
|
130
|
+
buildername='Experimental Builder',
|
131
|
+
slavename='somehost',
|
132
|
+
)
|
133
|
+
yield api.test('svn_mode') + api.properties(
|
134
|
+
mastername='experimental.svn',
|
135
|
+
buildername='Experimental SVN Builder',
|
136
|
+
slavename='somehost',
|
137
|
+
force=1
|
138
|
+
)
|
139
|
+
yield api.test('clobber') + api.properties(
|
140
|
+
mastername='experimental',
|
141
|
+
buildername='Experimental Builder',
|
142
|
+
slavename='somehost',
|
143
|
+
clobber=1
|
144
|
+
)
|
145
|
+
yield api.test('reset_root_solution_revision') + api.properties(
|
146
|
+
mastername='experimental',
|
147
|
+
buildername='Experimental Builder',
|
148
|
+
slavename='somehost',
|
149
|
+
root_solution_revision='revision',
|
150
|
+
)
|
151
|
+
yield api.test('gerrit_no_reset') + api.properties(
|
152
|
+
mastername='experimental',
|
153
|
+
buildername='Experimental Builder',
|
154
|
+
slavename='somehost',
|
155
|
+
gerrit_no_reset=1
|
156
|
+
)
|
157
|
+
yield api.test('tryjob_v8') + api.properties(
|
158
|
+
mastername='tryserver.chromium.linux',
|
159
|
+
buildername='linux_rel',
|
160
|
+
slavename='totallyaslave-c4',
|
161
|
+
issue=12345,
|
162
|
+
patchset=654321,
|
163
|
+
patch_url='http://src.chromium.org/foo/bar',
|
164
|
+
patch_project='v8',
|
165
|
+
revisions={'src/v8': 'abc'}
|
166
|
+
)
|
167
|
+
yield api.test('tryjob_v8_head_by_default') + api.properties.tryserver(
|
168
|
+
patch_project='v8',
|
169
|
+
)
|
170
|
+
yield api.test('tryjob_gerrit_angle') + api.properties.tryserver_gerrit(
|
171
|
+
full_project_name='angle/angle',
|
172
|
+
)
|
File without changes
|
@@ -0,0 +1,1764 @@
|
|
1
|
+
#!/usr/bin/env python
|
2
|
+
# Copyright 2014 The Chromium Authors. All rights reserved.
|
3
|
+
# Use of this source code is governed by a BSD-style license that can be
|
4
|
+
# found in the LICENSE file.
|
5
|
+
|
6
|
+
# TODO(hinoka): Use logging.
|
7
|
+
|
8
|
+
import cStringIO
|
9
|
+
import codecs
|
10
|
+
import collections
|
11
|
+
import copy
|
12
|
+
import ctypes
|
13
|
+
import json
|
14
|
+
import optparse
|
15
|
+
import os
|
16
|
+
import pprint
|
17
|
+
import random
|
18
|
+
import re
|
19
|
+
import socket
|
20
|
+
import subprocess
|
21
|
+
import sys
|
22
|
+
import tempfile
|
23
|
+
import threading
|
24
|
+
import time
|
25
|
+
import urllib2
|
26
|
+
import urlparse
|
27
|
+
import uuid
|
28
|
+
|
29
|
+
import os.path as path
|
30
|
+
|
31
|
+
# How many bytes at a time to read from pipes.
|
32
|
+
BUF_SIZE = 256
|
33
|
+
|
34
|
+
|
35
|
+
# TODO(luqui): This is a horrible hack to identify build_internal when build
|
36
|
+
# is a recipe dependency. bot_update should not be depending on internal,
|
37
|
+
# rather the arrow should go the other way (or just be destroyed).
|
38
|
+
def check_dir(name, dirs, default=None):
|
39
|
+
for d in dirs:
|
40
|
+
d = path.abspath(d)
|
41
|
+
if path.basename(d) == name and path.isdir(d):
|
42
|
+
return d
|
43
|
+
return default
|
44
|
+
|
45
|
+
|
46
|
+
# Define a bunch of directory paths.
|
47
|
+
# Relative to the current working directory.
|
48
|
+
CURRENT_DIR = path.abspath(os.getcwd())
|
49
|
+
BUILDER_DIR = path.dirname(CURRENT_DIR)
|
50
|
+
SLAVE_DIR = path.dirname(BUILDER_DIR)
|
51
|
+
|
52
|
+
# Relative to this script's filesystem path.
|
53
|
+
THIS_DIR = path.dirname(path.abspath(__file__))
|
54
|
+
SCRIPTS_DIR = check_dir(
|
55
|
+
'scripts', [
|
56
|
+
path.dirname(THIS_DIR),
|
57
|
+
path.join(SLAVE_DIR, '..', 'scripts'),
|
58
|
+
path.join(THIS_DIR, # resources
|
59
|
+
'..', # bot_update
|
60
|
+
'..', # recipe_modules
|
61
|
+
'..', # depot_tools
|
62
|
+
'..', # .recipe_deps
|
63
|
+
'..', # slave
|
64
|
+
'..', # scripts
|
65
|
+
'..', # build_internal
|
66
|
+
'..', # ROOT_DIR
|
67
|
+
'build',
|
68
|
+
'scripts'),
|
69
|
+
path.join(SLAVE_DIR, '..', 'build', 'scripts'),
|
70
|
+
], default=path.dirname(THIS_DIR))
|
71
|
+
BUILD_DIR = path.dirname(SCRIPTS_DIR)
|
72
|
+
ROOT_DIR = path.dirname(BUILD_DIR)
|
73
|
+
|
74
|
+
DEPOT_TOOLS_DIR = path.abspath(path.join(THIS_DIR, '..', '..', '..'))
|
75
|
+
|
76
|
+
BUILD_INTERNAL_DIR = check_dir(
|
77
|
+
'build_internal', [
|
78
|
+
path.join(ROOT_DIR, 'build_internal'),
|
79
|
+
path.join(ROOT_DIR, # .recipe_deps
|
80
|
+
path.pardir, # slave
|
81
|
+
path.pardir, # scripts
|
82
|
+
path.pardir), # build_internal
|
83
|
+
])
|
84
|
+
|
85
|
+
|
86
|
+
CHROMIUM_GIT_HOST = 'https://chromium.googlesource.com'
|
87
|
+
CHROMIUM_SRC_URL = CHROMIUM_GIT_HOST + '/chromium/src.git'
|
88
|
+
|
89
|
+
# Official builds use buildspecs, so this is a special case.
|
90
|
+
BUILDSPEC_TYPE = collections.namedtuple('buildspec',
|
91
|
+
('container', 'version'))
|
92
|
+
BUILDSPEC_RE = (r'^/chrome-internal/trunk/tools/buildspec/'
|
93
|
+
'(build|branches|releases)/(.+)$')
|
94
|
+
GIT_BUILDSPEC_PATH = ('https://chrome-internal.googlesource.com/chrome/tools/'
|
95
|
+
'buildspec')
|
96
|
+
BRANCH_HEADS_REFSPEC = '+refs/branch-heads/*'
|
97
|
+
|
98
|
+
BUILDSPEC_COMMIT_RE = (
|
99
|
+
re.compile(r'Buildspec for.*version (\d+\.\d+\.\d+\.\d+)'),
|
100
|
+
re.compile(r'Create (\d+\.\d+\.\d+\.\d+) buildspec'),
|
101
|
+
re.compile(r'Auto-converted (\d+\.\d+\.\d+\.\d+) buildspec to git'),
|
102
|
+
)
|
103
|
+
|
104
|
+
# Regular expression that matches a single commit footer line.
|
105
|
+
COMMIT_FOOTER_ENTRY_RE = re.compile(r'([^:]+):\s+(.+)')
|
106
|
+
|
107
|
+
# Footer metadata keys for regular and gsubtreed mirrored commit positions.
|
108
|
+
COMMIT_POSITION_FOOTER_KEY = 'Cr-Commit-Position'
|
109
|
+
COMMIT_ORIGINAL_POSITION_FOOTER_KEY = 'Cr-Original-Commit-Position'
|
110
|
+
# Regular expression to parse a commit position
|
111
|
+
COMMIT_POSITION_RE = re.compile(r'(.+)@\{#(\d+)\}')
|
112
|
+
|
113
|
+
# Regular expression to parse gclient's revinfo entries.
|
114
|
+
REVINFO_RE = re.compile(r'^([^:]+):\s+([^@]+)@(.+)$')
|
115
|
+
|
116
|
+
# Used by 'ResolveSvnRevisionFromGitiles'
|
117
|
+
GIT_SVN_PROJECT_MAP = {
|
118
|
+
'webkit': {
|
119
|
+
'svn_url': 'svn://svn.chromium.org/blink',
|
120
|
+
'branch_map': [
|
121
|
+
(r'trunk', r'refs/heads/master'),
|
122
|
+
(r'branches/([^/]+)', r'refs/branch-heads/\1'),
|
123
|
+
],
|
124
|
+
},
|
125
|
+
'v8': {
|
126
|
+
'svn_url': 'https://v8.googlecode.com/svn',
|
127
|
+
'branch_map': [
|
128
|
+
(r'trunk', r'refs/heads/candidates'),
|
129
|
+
(r'branches/bleeding_edge', r'refs/heads/master'),
|
130
|
+
(r'branches/([^/]+)', r'refs/branch-heads/\1'),
|
131
|
+
],
|
132
|
+
},
|
133
|
+
'nacl': {
|
134
|
+
'svn_url': 'svn://svn.chromium.org/native_client',
|
135
|
+
'branch_map': [
|
136
|
+
(r'trunk/src/native_client', r'refs/heads/master'),
|
137
|
+
],
|
138
|
+
},
|
139
|
+
}
|
140
|
+
|
141
|
+
# Key for the 'git-svn' ID metadata commit footer entry.
|
142
|
+
GIT_SVN_ID_FOOTER_KEY = 'git-svn-id'
|
143
|
+
# e.g., git-svn-id: https://v8.googlecode.com/svn/trunk@23117
|
144
|
+
# ce2b1a6d-e550-0410-aec6-3dcde31c8c00
|
145
|
+
GIT_SVN_ID_RE = re.compile(r'((?:\w+)://[^@]+)@(\d+)\s+(?:[a-zA-Z0-9\-]+)')
|
146
|
+
|
147
|
+
|
148
|
+
# This is the git mirror of the buildspecs repository. We could rely on the svn
|
149
|
+
# checkout, now that the git buildspecs are checked in alongside the svn
|
150
|
+
# buildspecs, but we're going to want to pull all the buildspecs from here
|
151
|
+
# eventually anyhow, and there's already some logic to pull from git (for the
|
152
|
+
# old git_buildspecs.git repo), so just stick with that.
|
153
|
+
GIT_BUILDSPEC_REPO = (
|
154
|
+
'https://chrome-internal.googlesource.com/chrome/tools/buildspec')
|
155
|
+
|
156
|
+
# Copied from scripts/recipes/chromium.py.
|
157
|
+
GOT_REVISION_MAPPINGS = {
|
158
|
+
'/chrome/trunk/src': {
|
159
|
+
'src/': 'got_revision',
|
160
|
+
'src/native_client/': 'got_nacl_revision',
|
161
|
+
'src/tools/swarm_client/': 'got_swarm_client_revision',
|
162
|
+
'src/tools/swarming_client/': 'got_swarming_client_revision',
|
163
|
+
'src/third_party/WebKit/': 'got_webkit_revision',
|
164
|
+
'src/third_party/webrtc/': 'got_webrtc_revision',
|
165
|
+
'src/v8/': 'got_v8_revision',
|
166
|
+
}
|
167
|
+
}
|
168
|
+
|
169
|
+
|
170
|
+
BOT_UPDATE_MESSAGE = """
|
171
|
+
What is the "Bot Update" step?
|
172
|
+
==============================
|
173
|
+
|
174
|
+
This step ensures that the source checkout on the bot (e.g. Chromium's src/ and
|
175
|
+
its dependencies) is checked out in a consistent state. This means that all of
|
176
|
+
the necessary repositories are checked out, no extra repositories are checked
|
177
|
+
out, and no locally modified files are present.
|
178
|
+
|
179
|
+
These actions used to be taken care of by the "gclient revert" and "update"
|
180
|
+
steps. However, those steps are known to be buggy and occasionally flaky. This
|
181
|
+
step has two main advantages over them:
|
182
|
+
* it only operates in Git, so the logic can be clearer and cleaner; and
|
183
|
+
* it is a slave-side script, so its behavior can be modified without
|
184
|
+
restarting the master.
|
185
|
+
|
186
|
+
Why Git, you ask? Because that is the direction that the Chromium project is
|
187
|
+
heading. This step is an integral part of the transition from using the SVN repo
|
188
|
+
at chrome/trunk/src to using the Git repo src.git. Please pardon the dust while
|
189
|
+
we fully convert everything to Git. This message will get out of your way
|
190
|
+
eventually, and the waterfall will be a happier place because of it.
|
191
|
+
|
192
|
+
This step can be activated or deactivated independently on every builder on
|
193
|
+
every master. When it is active, the "gclient revert" and "update" steps become
|
194
|
+
no-ops. When it is inactive, it prints this message, cleans up after itself, and
|
195
|
+
lets everything else continue as though nothing has changed. Eventually, when
|
196
|
+
everything is stable enough, this step will replace them entirely.
|
197
|
+
|
198
|
+
Debugging information:
|
199
|
+
(master/builder/slave may be unspecified on recipes)
|
200
|
+
master: %(master)s
|
201
|
+
builder: %(builder)s
|
202
|
+
slave: %(slave)s
|
203
|
+
forced by recipes: %(recipe)s
|
204
|
+
CURRENT_DIR: %(CURRENT_DIR)s
|
205
|
+
BUILDER_DIR: %(BUILDER_DIR)s
|
206
|
+
SLAVE_DIR: %(SLAVE_DIR)s
|
207
|
+
THIS_DIR: %(THIS_DIR)s
|
208
|
+
SCRIPTS_DIR: %(SCRIPTS_DIR)s
|
209
|
+
BUILD_DIR: %(BUILD_DIR)s
|
210
|
+
ROOT_DIR: %(ROOT_DIR)s
|
211
|
+
DEPOT_TOOLS_DIR: %(DEPOT_TOOLS_DIR)s
|
212
|
+
bot_update.py is:"""
|
213
|
+
|
214
|
+
ACTIVATED_MESSAGE = """ACTIVE.
|
215
|
+
The bot will perform a Git checkout in this step.
|
216
|
+
The "gclient revert" and "update" steps are no-ops.
|
217
|
+
|
218
|
+
"""
|
219
|
+
|
220
|
+
NOT_ACTIVATED_MESSAGE = """INACTIVE.
|
221
|
+
This step does nothing. You actually want to look at the "update" step.
|
222
|
+
|
223
|
+
"""
|
224
|
+
|
225
|
+
|
226
|
+
GCLIENT_TEMPLATE = """solutions = %(solutions)s
|
227
|
+
|
228
|
+
cache_dir = r%(cache_dir)s
|
229
|
+
%(target_os)s
|
230
|
+
%(target_os_only)s
|
231
|
+
"""
|
232
|
+
|
233
|
+
|
234
|
+
internal_data = {}
|
235
|
+
if BUILD_INTERNAL_DIR:
|
236
|
+
local_vars = {}
|
237
|
+
try:
|
238
|
+
execfile(os.path.join(
|
239
|
+
BUILD_INTERNAL_DIR, 'scripts', 'slave', 'bot_update_cfg.py'),
|
240
|
+
local_vars)
|
241
|
+
except Exception:
|
242
|
+
# Same as if BUILD_INTERNAL_DIR didn't exist in the first place.
|
243
|
+
print 'Warning: unable to read internal configuration file.'
|
244
|
+
print 'If this is an internal bot, this step may be erroneously inactive.'
|
245
|
+
internal_data = local_vars
|
246
|
+
|
247
|
+
RECOGNIZED_PATHS = {
|
248
|
+
# If SVN path matches key, the entire URL is rewritten to the Git url.
|
249
|
+
'/chrome/trunk/src':
|
250
|
+
CHROMIUM_SRC_URL,
|
251
|
+
'/chrome/trunk/src/tools/cros.DEPS':
|
252
|
+
CHROMIUM_GIT_HOST + '/chromium/src/tools/cros.DEPS.git',
|
253
|
+
}
|
254
|
+
RECOGNIZED_PATHS.update(internal_data.get('RECOGNIZED_PATHS', {}))
|
255
|
+
|
256
|
+
ENABLED_MASTERS = [
|
257
|
+
'bot_update.always_on',
|
258
|
+
'chromium.android',
|
259
|
+
'chromium.angle',
|
260
|
+
'chromium.chrome',
|
261
|
+
'chromium.chromedriver',
|
262
|
+
'chromium.chromiumos',
|
263
|
+
'chromium',
|
264
|
+
'chromium.fyi',
|
265
|
+
'chromium.goma',
|
266
|
+
'chromium.gpu',
|
267
|
+
'chromium.gpu.fyi',
|
268
|
+
'chromium.infra',
|
269
|
+
'chromium.infra.cron',
|
270
|
+
'chromium.linux',
|
271
|
+
'chromium.lkgr',
|
272
|
+
'chromium.mac',
|
273
|
+
'chromium.memory',
|
274
|
+
'chromium.memory.fyi',
|
275
|
+
'chromium.perf',
|
276
|
+
'chromium.perf.fyi',
|
277
|
+
'chromium.swarm',
|
278
|
+
'chromium.webkit',
|
279
|
+
'chromium.webrtc',
|
280
|
+
'chromium.webrtc.fyi',
|
281
|
+
'chromium.win',
|
282
|
+
'client.catapult',
|
283
|
+
'client.drmemory',
|
284
|
+
'client.mojo',
|
285
|
+
'client.nacl',
|
286
|
+
'client.nacl.ports',
|
287
|
+
'client.nacl.sdk',
|
288
|
+
'client.nacl.toolchain',
|
289
|
+
'client.pdfium',
|
290
|
+
'client.skia',
|
291
|
+
'client.skia.fyi',
|
292
|
+
'client.v8',
|
293
|
+
'client.v8.branches',
|
294
|
+
'client.v8.fyi',
|
295
|
+
'client.v8.ports',
|
296
|
+
'client.webrtc',
|
297
|
+
'client.webrtc.fyi',
|
298
|
+
'tryserver.blink',
|
299
|
+
'tryserver.client.catapult',
|
300
|
+
'tryserver.client.mojo',
|
301
|
+
'tryserver.chromium.android',
|
302
|
+
'tryserver.chromium.angle',
|
303
|
+
'tryserver.chromium.linux',
|
304
|
+
'tryserver.chromium.mac',
|
305
|
+
'tryserver.chromium.perf',
|
306
|
+
'tryserver.chromium.win',
|
307
|
+
'tryserver.infra',
|
308
|
+
'tryserver.nacl',
|
309
|
+
'tryserver.v8',
|
310
|
+
'tryserver.webrtc',
|
311
|
+
]
|
312
|
+
ENABLED_MASTERS += internal_data.get('ENABLED_MASTERS', [])
|
313
|
+
|
314
|
+
ENABLED_BUILDERS = {
|
315
|
+
'client.dart.fyi': [
|
316
|
+
'v8-linux-release',
|
317
|
+
'v8-mac-release',
|
318
|
+
'v8-win-release',
|
319
|
+
],
|
320
|
+
'client.dynamorio': [
|
321
|
+
'linux-v8-dr',
|
322
|
+
],
|
323
|
+
}
|
324
|
+
ENABLED_BUILDERS.update(internal_data.get('ENABLED_BUILDERS', {}))
|
325
|
+
|
326
|
+
ENABLED_SLAVES = {}
|
327
|
+
ENABLED_SLAVES.update(internal_data.get('ENABLED_SLAVES', {}))
|
328
|
+
|
329
|
+
# Disabled filters get run AFTER enabled filters, so for example if a builder
|
330
|
+
# config is enabled, but a bot on that builder is disabled, that bot will
|
331
|
+
# be disabled.
|
332
|
+
DISABLED_BUILDERS = {}
|
333
|
+
DISABLED_BUILDERS.update(internal_data.get('DISABLED_BUILDERS', {}))
|
334
|
+
|
335
|
+
DISABLED_SLAVES = {}
|
336
|
+
DISABLED_SLAVES.update(internal_data.get('DISABLED_SLAVES', {}))
|
337
|
+
|
338
|
+
# These masters work only in Git, meaning for got_revision, always output
|
339
|
+
# a git hash rather than a SVN rev.
|
340
|
+
GIT_MASTERS = [
|
341
|
+
'client.v8',
|
342
|
+
'client.v8.branches',
|
343
|
+
'client.v8.ports',
|
344
|
+
'tryserver.v8',
|
345
|
+
]
|
346
|
+
GIT_MASTERS += internal_data.get('GIT_MASTERS', [])
|
347
|
+
|
348
|
+
|
349
|
+
# How many times to try before giving up.
|
350
|
+
ATTEMPTS = 5
|
351
|
+
|
352
|
+
# Find deps2git
|
353
|
+
DEPS2GIT_DIR_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git')
|
354
|
+
DEPS2GIT_PATH = path.join(DEPS2GIT_DIR_PATH, 'deps2git.py')
|
355
|
+
S2G_INTERNAL_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git_internal',
|
356
|
+
'svn_to_git_internal.py')
|
357
|
+
GIT_CACHE_PATH = path.join(DEPOT_TOOLS_DIR, 'git_cache.py')
|
358
|
+
|
359
|
+
# Find the patch tool.
|
360
|
+
if sys.platform.startswith('win'):
|
361
|
+
if not BUILD_INTERNAL_DIR:
|
362
|
+
print 'Warning: could not find patch tool because there is no '
|
363
|
+
print 'build_internal present.'
|
364
|
+
PATCH_TOOL = None
|
365
|
+
else:
|
366
|
+
PATCH_TOOL = path.join(BUILD_INTERNAL_DIR, 'tools', 'patch.EXE')
|
367
|
+
else:
|
368
|
+
PATCH_TOOL = '/usr/bin/patch'
|
369
|
+
|
370
|
+
# If there is less than 100GB of disk space on the system, then we do
|
371
|
+
# a shallow checkout.
|
372
|
+
SHALLOW_CLONE_THRESHOLD = 100 * 1024 * 1024 * 1024
|
373
|
+
|
374
|
+
|
375
|
+
class SubprocessFailed(Exception):
|
376
|
+
def __init__(self, message, code, output):
|
377
|
+
Exception.__init__(self, message)
|
378
|
+
self.code = code
|
379
|
+
self.output = output
|
380
|
+
|
381
|
+
|
382
|
+
class PatchFailed(SubprocessFailed):
|
383
|
+
pass
|
384
|
+
|
385
|
+
|
386
|
+
class GclientSyncFailed(SubprocessFailed):
|
387
|
+
pass
|
388
|
+
|
389
|
+
|
390
|
+
class SVNRevisionNotFound(Exception):
|
391
|
+
pass
|
392
|
+
|
393
|
+
|
394
|
+
class InvalidDiff(Exception):
|
395
|
+
pass
|
396
|
+
|
397
|
+
|
398
|
+
class Inactive(Exception):
|
399
|
+
"""Not really an exception, just used to exit early cleanly."""
|
400
|
+
pass
|
401
|
+
|
402
|
+
|
403
|
+
RETRY = object()
|
404
|
+
OK = object()
|
405
|
+
FAIL = object()
|
406
|
+
|
407
|
+
|
408
|
+
class PsPrinter(object):
|
409
|
+
def __init__(self, interval=300):
|
410
|
+
self.interval = interval
|
411
|
+
self.active = sys.platform.startswith('linux2')
|
412
|
+
self.thread = None
|
413
|
+
|
414
|
+
@staticmethod
|
415
|
+
def print_pstree():
|
416
|
+
"""Debugging function used to print "ps auxwwf" for stuck processes."""
|
417
|
+
subprocess.call(['ps', 'auxwwf'])
|
418
|
+
|
419
|
+
def poke(self):
|
420
|
+
if self.active:
|
421
|
+
self.cancel()
|
422
|
+
self.thread = threading.Timer(self.interval, self.print_pstree)
|
423
|
+
self.thread.start()
|
424
|
+
|
425
|
+
def cancel(self):
|
426
|
+
if self.active and self.thread is not None:
|
427
|
+
self.thread.cancel()
|
428
|
+
self.thread = None
|
429
|
+
|
430
|
+
|
431
|
+
def call(*args, **kwargs): # pragma: no cover
|
432
|
+
"""Interactive subprocess call."""
|
433
|
+
kwargs['stdout'] = subprocess.PIPE
|
434
|
+
kwargs['stderr'] = subprocess.STDOUT
|
435
|
+
kwargs.setdefault('bufsize', BUF_SIZE)
|
436
|
+
cwd = kwargs.get('cwd', os.getcwd())
|
437
|
+
result_fn = kwargs.pop('result_fn', lambda code, out: RETRY if code else OK)
|
438
|
+
stdin_data = kwargs.pop('stdin_data', None)
|
439
|
+
tries = kwargs.pop('tries', ATTEMPTS)
|
440
|
+
if stdin_data:
|
441
|
+
kwargs['stdin'] = subprocess.PIPE
|
442
|
+
out = cStringIO.StringIO()
|
443
|
+
new_env = kwargs.get('env', {})
|
444
|
+
env = copy.copy(os.environ)
|
445
|
+
env.update(new_env)
|
446
|
+
kwargs['env'] = env
|
447
|
+
attempt = 0
|
448
|
+
for attempt in range(1, tries + 1):
|
449
|
+
attempt_msg = ' (attempt #%d)' % attempt if attempt else ''
|
450
|
+
if new_env:
|
451
|
+
print '===Injecting Environment Variables==='
|
452
|
+
for k, v in sorted(new_env.items()):
|
453
|
+
print '%s: %s' % (k, v)
|
454
|
+
print '===Running %s%s===' % (' '.join(args), attempt_msg)
|
455
|
+
print 'In directory: %s' % cwd
|
456
|
+
start_time = time.time()
|
457
|
+
proc = subprocess.Popen(args, **kwargs)
|
458
|
+
if stdin_data:
|
459
|
+
proc.stdin.write(stdin_data)
|
460
|
+
proc.stdin.close()
|
461
|
+
psprinter = PsPrinter()
|
462
|
+
# This is here because passing 'sys.stdout' into stdout for proc will
|
463
|
+
# produce out of order output.
|
464
|
+
hanging_cr = False
|
465
|
+
while True:
|
466
|
+
psprinter.poke()
|
467
|
+
buf = proc.stdout.read(BUF_SIZE)
|
468
|
+
if not buf:
|
469
|
+
break
|
470
|
+
if hanging_cr:
|
471
|
+
buf = '\r' + buf
|
472
|
+
hanging_cr = buf.endswith('\r')
|
473
|
+
if hanging_cr:
|
474
|
+
buf = buf[:-1]
|
475
|
+
buf = buf.replace('\r\n', '\n').replace('\r', '\n')
|
476
|
+
sys.stdout.write(buf)
|
477
|
+
out.write(buf)
|
478
|
+
if hanging_cr:
|
479
|
+
sys.stdout.write('\n')
|
480
|
+
out.write('\n')
|
481
|
+
psprinter.cancel()
|
482
|
+
|
483
|
+
code = proc.wait()
|
484
|
+
elapsed_time = ((time.time() - start_time) / 60.0)
|
485
|
+
outval = out.getvalue()
|
486
|
+
result = result_fn(code, outval)
|
487
|
+
if result in (FAIL, RETRY):
|
488
|
+
print '===Failed in %.1f mins===' % elapsed_time
|
489
|
+
print
|
490
|
+
else:
|
491
|
+
print '===Succeeded in %.1f mins===' % elapsed_time
|
492
|
+
print
|
493
|
+
return outval
|
494
|
+
if result is FAIL:
|
495
|
+
break
|
496
|
+
if result is RETRY and attempt < tries:
|
497
|
+
sleep_backoff = 4 ** attempt
|
498
|
+
sleep_time = random.randint(sleep_backoff, int(sleep_backoff * 1.2))
|
499
|
+
print '===backing off, sleeping for %d secs===' % sleep_time
|
500
|
+
time.sleep(sleep_time)
|
501
|
+
|
502
|
+
raise SubprocessFailed('%s failed with code %d in %s after %d attempts.' %
|
503
|
+
(' '.join(args), code, cwd, attempt),
|
504
|
+
code, outval)
|
505
|
+
|
506
|
+
|
507
|
+
def git(*args, **kwargs): # pragma: no cover
|
508
|
+
"""Wrapper around call specifically for Git commands."""
|
509
|
+
if args and args[0] == 'cache':
|
510
|
+
# Rewrite "git cache" calls into "python git_cache.py".
|
511
|
+
cmd = (sys.executable, '-u', GIT_CACHE_PATH) + args[1:]
|
512
|
+
else:
|
513
|
+
git_executable = 'git'
|
514
|
+
# On windows, subprocess doesn't fuzzy-match 'git' to 'git.bat', so we
|
515
|
+
# have to do it explicitly. This is better than passing shell=True.
|
516
|
+
if sys.platform.startswith('win'):
|
517
|
+
git_executable += '.bat'
|
518
|
+
cmd = (git_executable,) + args
|
519
|
+
return call(*cmd, **kwargs)
|
520
|
+
|
521
|
+
|
522
|
+
def get_gclient_spec(solutions, target_os, target_os_only, git_cache_dir):
|
523
|
+
return GCLIENT_TEMPLATE % {
|
524
|
+
'solutions': pprint.pformat(solutions, indent=4),
|
525
|
+
'cache_dir': '"%s"' % git_cache_dir,
|
526
|
+
'target_os': ('\ntarget_os=%s' % target_os) if target_os else '',
|
527
|
+
'target_os_only': '\ntarget_os_only=%s' % target_os_only
|
528
|
+
}
|
529
|
+
|
530
|
+
|
531
|
+
def check_enabled(master, builder, slave):
|
532
|
+
if master in ENABLED_MASTERS:
|
533
|
+
return True
|
534
|
+
builder_list = ENABLED_BUILDERS.get(master)
|
535
|
+
if builder_list and builder in builder_list:
|
536
|
+
return True
|
537
|
+
slave_list = ENABLED_SLAVES.get(master)
|
538
|
+
if slave_list and slave in slave_list:
|
539
|
+
return True
|
540
|
+
return False
|
541
|
+
|
542
|
+
|
543
|
+
def check_disabled(master, builder, slave):
|
544
|
+
"""Returns True if disabled, False if not disabled."""
|
545
|
+
builder_list = DISABLED_BUILDERS.get(master)
|
546
|
+
if builder_list and builder in builder_list:
|
547
|
+
return True
|
548
|
+
slave_list = DISABLED_SLAVES.get(master)
|
549
|
+
if slave_list and slave in slave_list:
|
550
|
+
return True
|
551
|
+
return False
|
552
|
+
|
553
|
+
|
554
|
+
def check_valid_host(master, builder, slave):
|
555
|
+
return (check_enabled(master, builder, slave)
|
556
|
+
and not check_disabled(master, builder, slave))
|
557
|
+
|
558
|
+
|
559
|
+
def maybe_ignore_revision(revision, buildspec):
|
560
|
+
"""Handle builders that don't care what buildbot tells them to build.
|
561
|
+
|
562
|
+
This is especially the case with branch builders that build from buildspecs
|
563
|
+
and/or trigger off multiple repositories, where the --revision passed in has
|
564
|
+
nothing to do with the solution being built. Clearing the revision in this
|
565
|
+
case causes bot_update to use HEAD rather that trying to checkout an
|
566
|
+
inappropriate version of the solution.
|
567
|
+
"""
|
568
|
+
if buildspec and buildspec.container == 'branches':
|
569
|
+
return []
|
570
|
+
return revision
|
571
|
+
|
572
|
+
|
573
|
+
def solutions_printer(solutions):
|
574
|
+
"""Prints gclient solution to stdout."""
|
575
|
+
print 'Gclient Solutions'
|
576
|
+
print '================='
|
577
|
+
for solution in solutions:
|
578
|
+
name = solution.get('name')
|
579
|
+
url = solution.get('url')
|
580
|
+
print '%s (%s)' % (name, url)
|
581
|
+
if solution.get('deps_file'):
|
582
|
+
print ' Dependencies file is %s' % solution['deps_file']
|
583
|
+
if 'managed' in solution:
|
584
|
+
print ' Managed mode is %s' % ('ON' if solution['managed'] else 'OFF')
|
585
|
+
custom_vars = solution.get('custom_vars')
|
586
|
+
if custom_vars:
|
587
|
+
print ' Custom Variables:'
|
588
|
+
for var_name, var_value in sorted(custom_vars.iteritems()):
|
589
|
+
print ' %s = %s' % (var_name, var_value)
|
590
|
+
custom_deps = solution.get('custom_deps')
|
591
|
+
if 'custom_deps' in solution:
|
592
|
+
print ' Custom Dependencies:'
|
593
|
+
for deps_name, deps_value in sorted(custom_deps.iteritems()):
|
594
|
+
if deps_value:
|
595
|
+
print ' %s -> %s' % (deps_name, deps_value)
|
596
|
+
else:
|
597
|
+
print ' %s: Ignore' % deps_name
|
598
|
+
for k, v in solution.iteritems():
|
599
|
+
# Print out all the keys we don't know about.
|
600
|
+
if k in ['name', 'url', 'deps_file', 'custom_vars', 'custom_deps',
|
601
|
+
'managed']:
|
602
|
+
continue
|
603
|
+
print ' %s is %s' % (k, v)
|
604
|
+
print
|
605
|
+
|
606
|
+
|
607
|
+
def solutions_to_git(input_solutions):
|
608
|
+
"""Modifies urls in solutions to point at Git repos.
|
609
|
+
|
610
|
+
returns: (git solution, svn root of first solution) tuple.
|
611
|
+
"""
|
612
|
+
assert input_solutions
|
613
|
+
solutions = copy.deepcopy(input_solutions)
|
614
|
+
first_solution = True
|
615
|
+
buildspec = None
|
616
|
+
for solution in solutions:
|
617
|
+
original_url = solution['url']
|
618
|
+
parsed_url = urlparse.urlparse(original_url)
|
619
|
+
parsed_path = parsed_url.path
|
620
|
+
|
621
|
+
# Rewrite SVN urls into Git urls.
|
622
|
+
buildspec_m = re.match(BUILDSPEC_RE, parsed_path)
|
623
|
+
if first_solution and buildspec_m:
|
624
|
+
solution['url'] = GIT_BUILDSPEC_PATH
|
625
|
+
buildspec = BUILDSPEC_TYPE(
|
626
|
+
container=buildspec_m.group(1),
|
627
|
+
version=buildspec_m.group(2),
|
628
|
+
)
|
629
|
+
solution['deps_file'] = path.join(buildspec.container, buildspec.version,
|
630
|
+
'DEPS')
|
631
|
+
elif parsed_path in RECOGNIZED_PATHS:
|
632
|
+
solution['url'] = RECOGNIZED_PATHS[parsed_path]
|
633
|
+
solution['deps_file'] = '.DEPS.git'
|
634
|
+
elif parsed_url.scheme == 'https' and 'googlesource' in parsed_url.netloc:
|
635
|
+
pass
|
636
|
+
else:
|
637
|
+
print 'Warning: %s' % ('path %r not recognized' % parsed_path,)
|
638
|
+
|
639
|
+
# Strip out deps containing $$V8_REV$$, etc.
|
640
|
+
if 'custom_deps' in solution:
|
641
|
+
new_custom_deps = {}
|
642
|
+
for deps_name, deps_value in solution['custom_deps'].iteritems():
|
643
|
+
if deps_value and '$$' in deps_value:
|
644
|
+
print 'Dropping %s:%s from custom deps' % (deps_name, deps_value)
|
645
|
+
else:
|
646
|
+
new_custom_deps[deps_name] = deps_value
|
647
|
+
solution['custom_deps'] = new_custom_deps
|
648
|
+
|
649
|
+
if first_solution:
|
650
|
+
root = parsed_path
|
651
|
+
first_solution = False
|
652
|
+
|
653
|
+
solution['managed'] = False
|
654
|
+
# We don't want gclient to be using a safesync URL. Instead it should
|
655
|
+
# using the lkgr/lkcr branch/tags.
|
656
|
+
if 'safesync_url' in solution:
|
657
|
+
print 'Removing safesync url %s from %s' % (solution['safesync_url'],
|
658
|
+
parsed_path)
|
659
|
+
del solution['safesync_url']
|
660
|
+
return solutions, root, buildspec
|
661
|
+
|
662
|
+
|
663
|
+
def remove(target):
|
664
|
+
"""Remove a target by moving it into build.dead."""
|
665
|
+
dead_folder = path.join(BUILDER_DIR, 'build.dead')
|
666
|
+
if not path.exists(dead_folder):
|
667
|
+
os.makedirs(dead_folder)
|
668
|
+
os.rename(target, path.join(dead_folder, uuid.uuid4().hex))
|
669
|
+
|
670
|
+
|
671
|
+
def ensure_no_checkout(dir_names, scm_dirname):
|
672
|
+
"""Ensure that there is no undesired checkout under build/.
|
673
|
+
|
674
|
+
If there is an incorrect checkout under build/, then
|
675
|
+
move build/ to build.dead/
|
676
|
+
This function will check each directory in dir_names.
|
677
|
+
|
678
|
+
scm_dirname is expected to be either ['.svn', '.git']
|
679
|
+
"""
|
680
|
+
assert scm_dirname in ['.svn', '.git', '*']
|
681
|
+
has_checkout = any(path.exists(path.join(os.getcwd(), dir_name, scm_dirname))
|
682
|
+
for dir_name in dir_names)
|
683
|
+
|
684
|
+
if has_checkout or scm_dirname == '*':
|
685
|
+
build_dir = os.getcwd()
|
686
|
+
prefix = ''
|
687
|
+
if scm_dirname != '*':
|
688
|
+
prefix = '%s detected in checkout, ' % scm_dirname
|
689
|
+
|
690
|
+
for filename in os.listdir(build_dir):
|
691
|
+
deletion_target = path.join(build_dir, filename)
|
692
|
+
print '%sdeleting %s...' % (prefix, deletion_target),
|
693
|
+
remove(deletion_target)
|
694
|
+
print 'done'
|
695
|
+
|
696
|
+
|
697
|
+
def gclient_configure(solutions, target_os, target_os_only, git_cache_dir):
|
698
|
+
"""Should do the same thing as gclient --spec='...'."""
|
699
|
+
with codecs.open('.gclient', mode='w', encoding='utf-8') as f:
|
700
|
+
f.write(get_gclient_spec(
|
701
|
+
solutions, target_os, target_os_only, git_cache_dir))
|
702
|
+
|
703
|
+
|
704
|
+
def gclient_sync(with_branch_heads, shallow):
|
705
|
+
# We just need to allocate a filename.
|
706
|
+
fd, gclient_output_file = tempfile.mkstemp(suffix='.json')
|
707
|
+
os.close(fd)
|
708
|
+
gclient_bin = 'gclient.bat' if sys.platform.startswith('win') else 'gclient'
|
709
|
+
cmd = [gclient_bin, 'sync', '--verbose', '--reset', '--force',
|
710
|
+
'--ignore_locks', '--output-json', gclient_output_file,
|
711
|
+
'--nohooks', '--noprehooks', '--delete_unversioned_trees']
|
712
|
+
if with_branch_heads:
|
713
|
+
cmd += ['--with_branch_heads']
|
714
|
+
if shallow:
|
715
|
+
cmd += ['--shallow']
|
716
|
+
|
717
|
+
try:
|
718
|
+
call(*cmd, tries=1)
|
719
|
+
except SubprocessFailed as e:
|
720
|
+
# Throw a GclientSyncFailed exception so we can catch this independently.
|
721
|
+
raise GclientSyncFailed(e.message, e.code, e.output)
|
722
|
+
else:
|
723
|
+
with open(gclient_output_file) as f:
|
724
|
+
return json.load(f)
|
725
|
+
finally:
|
726
|
+
os.remove(gclient_output_file)
|
727
|
+
|
728
|
+
|
729
|
+
def gclient_runhooks(gyp_envs):
|
730
|
+
gclient_bin = 'gclient.bat' if sys.platform.startswith('win') else 'gclient'
|
731
|
+
env = dict([env_var.split('=', 1) for env_var in gyp_envs])
|
732
|
+
call(gclient_bin, 'runhooks', env=env)
|
733
|
+
|
734
|
+
|
735
|
+
def gclient_revinfo():
|
736
|
+
gclient_bin = 'gclient.bat' if sys.platform.startswith('win') else 'gclient'
|
737
|
+
return call(gclient_bin, 'revinfo', '-a') or ''
|
738
|
+
|
739
|
+
|
740
|
+
def create_manifest():
|
741
|
+
manifest = {}
|
742
|
+
output = gclient_revinfo()
|
743
|
+
for line in output.strip().splitlines():
|
744
|
+
match = REVINFO_RE.match(line.strip())
|
745
|
+
if match:
|
746
|
+
manifest[match.group(1)] = {
|
747
|
+
'repository': match.group(2),
|
748
|
+
'revision': match.group(3),
|
749
|
+
}
|
750
|
+
else:
|
751
|
+
print "WARNING: Couldn't match revinfo line:\n%s" % line
|
752
|
+
return manifest
|
753
|
+
|
754
|
+
|
755
|
+
def get_commit_message_footer_map(message):
|
756
|
+
"""Returns: (dict) A dictionary of commit message footer entries.
|
757
|
+
"""
|
758
|
+
footers = {}
|
759
|
+
|
760
|
+
# Extract the lines in the footer block.
|
761
|
+
lines = []
|
762
|
+
for line in message.strip().splitlines():
|
763
|
+
line = line.strip()
|
764
|
+
if len(line) == 0:
|
765
|
+
del lines[:]
|
766
|
+
continue
|
767
|
+
lines.append(line)
|
768
|
+
|
769
|
+
# Parse the footer
|
770
|
+
for line in lines:
|
771
|
+
m = COMMIT_FOOTER_ENTRY_RE.match(line)
|
772
|
+
if not m:
|
773
|
+
# If any single line isn't valid, the entire footer is invalid.
|
774
|
+
footers.clear()
|
775
|
+
return footers
|
776
|
+
footers[m.group(1)] = m.group(2).strip()
|
777
|
+
return footers
|
778
|
+
|
779
|
+
|
780
|
+
def get_commit_message_footer(message, key):
|
781
|
+
"""Returns: (str/None) The footer value for 'key', or None if none was found.
|
782
|
+
"""
|
783
|
+
return get_commit_message_footer_map(message).get(key)
|
784
|
+
|
785
|
+
|
786
|
+
def get_svn_rev(git_hash, dir_name):
|
787
|
+
log = git('log', '-1', git_hash, cwd=dir_name)
|
788
|
+
git_svn_id = get_commit_message_footer(log, GIT_SVN_ID_FOOTER_KEY)
|
789
|
+
if not git_svn_id:
|
790
|
+
return None
|
791
|
+
m = GIT_SVN_ID_RE.match(git_svn_id)
|
792
|
+
if not m:
|
793
|
+
return None
|
794
|
+
return int(m.group(2))
|
795
|
+
|
796
|
+
|
797
|
+
def get_git_hash(revision, branch, sln_dir):
|
798
|
+
"""We want to search for the SVN revision on the git-svn branch.
|
799
|
+
|
800
|
+
Note that git will search backwards from origin/master.
|
801
|
+
"""
|
802
|
+
match = "^%s: [^ ]*@%s " % (GIT_SVN_ID_FOOTER_KEY, revision)
|
803
|
+
ref = branch if branch.startswith('refs/') else 'origin/%s' % branch
|
804
|
+
cmd = ['log', '-E', '--grep', match, '--format=%H', '--max-count=1', ref]
|
805
|
+
result = git(*cmd, cwd=sln_dir).strip()
|
806
|
+
if result:
|
807
|
+
return result
|
808
|
+
raise SVNRevisionNotFound('We can\'t resolve svn r%s into a git hash in %s' %
|
809
|
+
(revision, sln_dir))
|
810
|
+
|
811
|
+
|
812
|
+
def _last_commit_for_file(filename, repo_base):
|
813
|
+
cmd = ['log', '--format=%H', '--max-count=1', '--', filename]
|
814
|
+
return git(*cmd, cwd=repo_base).strip()
|
815
|
+
|
816
|
+
|
817
|
+
def need_to_run_deps2git(repo_base, deps_file, deps_git_file):
|
818
|
+
"""Checks to see if we need to run deps2git.
|
819
|
+
|
820
|
+
Returns True if there was a DEPS change after the last .DEPS.git update
|
821
|
+
or if DEPS has local modifications.
|
822
|
+
"""
|
823
|
+
# See if DEPS is dirty
|
824
|
+
deps_file_status = git(
|
825
|
+
'status', '--porcelain', deps_file, cwd=repo_base).strip()
|
826
|
+
if deps_file_status and deps_file_status.startswith('M '):
|
827
|
+
return True
|
828
|
+
|
829
|
+
last_known_deps_ref = _last_commit_for_file(deps_file, repo_base)
|
830
|
+
last_known_deps_git_ref = _last_commit_for_file(deps_git_file, repo_base)
|
831
|
+
merge_base_ref = git('merge-base', last_known_deps_ref,
|
832
|
+
last_known_deps_git_ref, cwd=repo_base).strip()
|
833
|
+
|
834
|
+
# If the merge base of the last DEPS and last .DEPS.git file is not
|
835
|
+
# equivilent to the hash of the last DEPS file, that means the DEPS file
|
836
|
+
# was committed after the last .DEPS.git file.
|
837
|
+
return last_known_deps_ref != merge_base_ref
|
838
|
+
|
839
|
+
|
840
|
+
def ensure_deps2git(solution, shallow, git_cache_dir):
|
841
|
+
repo_base = path.join(os.getcwd(), solution['name'])
|
842
|
+
deps_file = path.join(repo_base, 'DEPS')
|
843
|
+
deps_git_file = path.join(repo_base, '.DEPS.git')
|
844
|
+
if (not git('ls-files', 'DEPS', cwd=repo_base).strip() or
|
845
|
+
not git('ls-files', '.DEPS.git', cwd=repo_base).strip()):
|
846
|
+
return
|
847
|
+
|
848
|
+
print 'Checking if %s is newer than %s' % (deps_file, deps_git_file)
|
849
|
+
if not need_to_run_deps2git(repo_base, deps_file, deps_git_file):
|
850
|
+
return
|
851
|
+
|
852
|
+
print '===DEPS file modified, need to run deps2git==='
|
853
|
+
cmd = [sys.executable, DEPS2GIT_PATH,
|
854
|
+
'--workspace', os.getcwd(),
|
855
|
+
'--cache_dir', git_cache_dir,
|
856
|
+
'--deps', deps_file,
|
857
|
+
'--out', deps_git_file]
|
858
|
+
if 'chrome-internal.googlesource' in solution['url']:
|
859
|
+
cmd.extend(['--extra-rules', S2G_INTERNAL_PATH])
|
860
|
+
if shallow:
|
861
|
+
cmd.append('--shallow')
|
862
|
+
call(*cmd)
|
863
|
+
|
864
|
+
|
865
|
+
def emit_log_lines(name, lines):
|
866
|
+
for line in lines.splitlines():
|
867
|
+
print '@@@STEP_LOG_LINE@%s@%s@@@' % (name, line)
|
868
|
+
print '@@@STEP_LOG_END@%s@@@' % name
|
869
|
+
|
870
|
+
|
871
|
+
def emit_properties(properties):
|
872
|
+
for property_name, property_value in sorted(properties.items()):
|
873
|
+
print '@@@SET_BUILD_PROPERTY@%s@"%s"@@@' % (property_name, property_value)
|
874
|
+
|
875
|
+
|
876
|
+
# Derived from:
|
877
|
+
# http://code.activestate.com/recipes/577972-disk-usage/?in=user-4178764
|
878
|
+
def get_total_disk_space():
|
879
|
+
cwd = os.getcwd()
|
880
|
+
# Windows is the only platform that doesn't support os.statvfs, so
|
881
|
+
# we need to special case this.
|
882
|
+
if sys.platform.startswith('win'):
|
883
|
+
_, total, free = (ctypes.c_ulonglong(), ctypes.c_ulonglong(), \
|
884
|
+
ctypes.c_ulonglong())
|
885
|
+
if sys.version_info >= (3,) or isinstance(cwd, unicode):
|
886
|
+
fn = ctypes.windll.kernel32.GetDiskFreeSpaceExW
|
887
|
+
else:
|
888
|
+
fn = ctypes.windll.kernel32.GetDiskFreeSpaceExA
|
889
|
+
ret = fn(cwd, ctypes.byref(_), ctypes.byref(total), ctypes.byref(free))
|
890
|
+
if ret == 0:
|
891
|
+
# WinError() will fetch the last error code.
|
892
|
+
raise ctypes.WinError()
|
893
|
+
return (total.value, free.value)
|
894
|
+
|
895
|
+
else:
|
896
|
+
st = os.statvfs(cwd)
|
897
|
+
free = st.f_bavail * st.f_frsize
|
898
|
+
total = st.f_blocks * st.f_frsize
|
899
|
+
return (total, free)
|
900
|
+
|
901
|
+
|
902
|
+
def get_target_revision(folder_name, git_url, revisions):
|
903
|
+
normalized_name = folder_name.strip('/')
|
904
|
+
if normalized_name in revisions:
|
905
|
+
return revisions[normalized_name]
|
906
|
+
if git_url in revisions:
|
907
|
+
return revisions[git_url]
|
908
|
+
return None
|
909
|
+
|
910
|
+
|
911
|
+
def force_revision(folder_name, revision):
|
912
|
+
split_revision = revision.split(':', 1)
|
913
|
+
branch = 'master'
|
914
|
+
if len(split_revision) == 2:
|
915
|
+
# Support for "branch:revision" syntax.
|
916
|
+
branch, revision = split_revision
|
917
|
+
|
918
|
+
if revision and revision.upper() != 'HEAD':
|
919
|
+
if revision and revision.isdigit() and len(revision) < 40:
|
920
|
+
# rev_num is really a svn revision number, convert it into a git hash.
|
921
|
+
git_ref = get_git_hash(int(revision), branch, folder_name)
|
922
|
+
else:
|
923
|
+
# rev_num is actually a git hash or ref, we can just use it.
|
924
|
+
git_ref = revision
|
925
|
+
git('checkout', '--force', git_ref, cwd=folder_name)
|
926
|
+
else:
|
927
|
+
ref = branch if branch.startswith('refs/') else 'origin/%s' % branch
|
928
|
+
git('checkout', '--force', ref, cwd=folder_name)
|
929
|
+
|
930
|
+
def git_checkout(solutions, revisions, shallow, refs, git_cache_dir):
|
931
|
+
build_dir = os.getcwd()
|
932
|
+
# Before we do anything, break all git_cache locks.
|
933
|
+
if path.isdir(git_cache_dir):
|
934
|
+
git('cache', 'unlock', '-vv', '--force', '--all',
|
935
|
+
'--cache-dir', git_cache_dir)
|
936
|
+
for item in os.listdir(git_cache_dir):
|
937
|
+
filename = os.path.join(git_cache_dir, item)
|
938
|
+
if item.endswith('.lock'):
|
939
|
+
raise Exception('%s exists after cache unlock' % filename)
|
940
|
+
first_solution = True
|
941
|
+
for sln in solutions:
|
942
|
+
# This is so we can loop back and try again if we need to wait for the
|
943
|
+
# git mirrors to update from SVN.
|
944
|
+
done = False
|
945
|
+
tries_left = 60
|
946
|
+
while not done:
|
947
|
+
name = sln['name']
|
948
|
+
url = sln['url']
|
949
|
+
if url == CHROMIUM_SRC_URL or url + '.git' == CHROMIUM_SRC_URL:
|
950
|
+
# Experiments show there's little to be gained from
|
951
|
+
# a shallow clone of src.
|
952
|
+
shallow = False
|
953
|
+
sln_dir = path.join(build_dir, name)
|
954
|
+
s = ['--shallow'] if shallow else []
|
955
|
+
populate_cmd = (['cache', 'populate', '--ignore_locks', '-v',
|
956
|
+
'--cache-dir', git_cache_dir] + s + [url])
|
957
|
+
for ref in refs:
|
958
|
+
populate_cmd.extend(['--ref', ref])
|
959
|
+
git(*populate_cmd)
|
960
|
+
mirror_dir = git(
|
961
|
+
'cache', 'exists', '--quiet',
|
962
|
+
'--cache-dir', git_cache_dir, url).strip()
|
963
|
+
clone_cmd = (
|
964
|
+
'clone', '--no-checkout', '--local', '--shared', mirror_dir, sln_dir)
|
965
|
+
|
966
|
+
try:
|
967
|
+
if not path.isdir(sln_dir):
|
968
|
+
git(*clone_cmd)
|
969
|
+
else:
|
970
|
+
git('remote', 'set-url', 'origin', mirror_dir, cwd=sln_dir)
|
971
|
+
git('fetch', 'origin', cwd=sln_dir)
|
972
|
+
for ref in refs:
|
973
|
+
refspec = '%s:%s' % (ref, ref.lstrip('+'))
|
974
|
+
git('fetch', 'origin', refspec, cwd=sln_dir)
|
975
|
+
|
976
|
+
revision = get_target_revision(name, url, revisions) or 'HEAD'
|
977
|
+
force_revision(sln_dir, revision)
|
978
|
+
done = True
|
979
|
+
except SubprocessFailed as e:
|
980
|
+
# Exited abnormally, theres probably something wrong.
|
981
|
+
# Lets wipe the checkout and try again.
|
982
|
+
tries_left -= 1
|
983
|
+
if tries_left > 0:
|
984
|
+
print 'Something failed: %s.' % str(e)
|
985
|
+
print 'waiting 5 seconds and trying again...'
|
986
|
+
time.sleep(5)
|
987
|
+
else:
|
988
|
+
raise
|
989
|
+
remove(sln_dir)
|
990
|
+
except SVNRevisionNotFound:
|
991
|
+
tries_left -= 1
|
992
|
+
if tries_left > 0:
|
993
|
+
# If we don't have the correct revision, wait and try again.
|
994
|
+
print 'We can\'t find revision %s.' % revision
|
995
|
+
print 'The svn to git replicator is probably falling behind.'
|
996
|
+
print 'waiting 5 seconds and trying again...'
|
997
|
+
time.sleep(5)
|
998
|
+
else:
|
999
|
+
raise
|
1000
|
+
|
1001
|
+
git('clean', '-dff', cwd=sln_dir)
|
1002
|
+
|
1003
|
+
if first_solution:
|
1004
|
+
git_ref = git('log', '--format=%H', '--max-count=1',
|
1005
|
+
cwd=sln_dir).strip()
|
1006
|
+
first_solution = False
|
1007
|
+
return git_ref
|
1008
|
+
|
1009
|
+
|
1010
|
+
def _download(url):
|
1011
|
+
"""Fetch url and return content, with retries for flake."""
|
1012
|
+
for attempt in xrange(ATTEMPTS):
|
1013
|
+
try:
|
1014
|
+
return urllib2.urlopen(url).read()
|
1015
|
+
except Exception:
|
1016
|
+
if attempt == ATTEMPTS - 1:
|
1017
|
+
raise
|
1018
|
+
|
1019
|
+
|
1020
|
+
def parse_diff(diff):
|
1021
|
+
"""Takes a unified diff and returns a list of diffed files and their diffs.
|
1022
|
+
|
1023
|
+
The return format is a list of pairs of:
|
1024
|
+
(<filename>, <diff contents>)
|
1025
|
+
<diff contents> is inclusive of the diff line.
|
1026
|
+
"""
|
1027
|
+
result = []
|
1028
|
+
current_diff = ''
|
1029
|
+
current_header = None
|
1030
|
+
for line in diff.splitlines():
|
1031
|
+
# "diff" is for git style patches, and "Index: " is for SVN style patches.
|
1032
|
+
if line.startswith('diff') or line.startswith('Index: '):
|
1033
|
+
if current_header:
|
1034
|
+
# If we are in a diff portion, then save the diff.
|
1035
|
+
result.append((current_header, '%s\n' % current_diff))
|
1036
|
+
git_header_match = re.match(r'diff (?:--git )?(\S+) (\S+)', line)
|
1037
|
+
svn_header_match = re.match(r'Index: (.*)', line)
|
1038
|
+
|
1039
|
+
if git_header_match:
|
1040
|
+
# First, see if its a git style header.
|
1041
|
+
from_file = git_header_match.group(1)
|
1042
|
+
to_file = git_header_match.group(2)
|
1043
|
+
if from_file != to_file and from_file.startswith('a/'):
|
1044
|
+
# Sometimes git prepends 'a/' and 'b/' in front of file paths.
|
1045
|
+
from_file = from_file[2:]
|
1046
|
+
current_header = from_file
|
1047
|
+
|
1048
|
+
elif svn_header_match:
|
1049
|
+
# Otherwise, check if its an SVN style header.
|
1050
|
+
current_header = svn_header_match.group(1)
|
1051
|
+
|
1052
|
+
else:
|
1053
|
+
# Otherwise... I'm not really sure what to do with this.
|
1054
|
+
raise InvalidDiff('Can\'t process header: %s\nFull diff:\n%s' %
|
1055
|
+
(line, diff))
|
1056
|
+
|
1057
|
+
current_diff = ''
|
1058
|
+
current_diff += '%s\n' % line
|
1059
|
+
if current_header:
|
1060
|
+
# We hit EOF, gotta save the last diff.
|
1061
|
+
result.append((current_header, current_diff))
|
1062
|
+
return result
|
1063
|
+
|
1064
|
+
|
1065
|
+
def get_svn_patch(patch_url):
|
1066
|
+
"""Fetch patch from patch_url, return list of (filename, diff)"""
|
1067
|
+
svn_exe = 'svn.bat' if sys.platform.startswith('win') else 'svn'
|
1068
|
+
patch_data = call(svn_exe, 'cat', patch_url)
|
1069
|
+
return parse_diff(patch_data)
|
1070
|
+
|
1071
|
+
|
1072
|
+
def apply_svn_patch(patch_root, patches, whitelist=None, blacklist=None):
|
1073
|
+
"""Expects a list of (filename, diff), applies it on top of patch_root."""
|
1074
|
+
if whitelist:
|
1075
|
+
patches = [(name, diff) for name, diff in patches if name in whitelist]
|
1076
|
+
elif blacklist:
|
1077
|
+
patches = [(name, diff) for name, diff in patches if name not in blacklist]
|
1078
|
+
diffs = [diff for _, diff in patches]
|
1079
|
+
patch = ''.join(diffs)
|
1080
|
+
|
1081
|
+
if patch:
|
1082
|
+
print '===Patching files==='
|
1083
|
+
for filename, _ in patches:
|
1084
|
+
print 'Patching %s' % filename
|
1085
|
+
try:
|
1086
|
+
call(PATCH_TOOL, '-p0', '--remove-empty-files', '--force', '--forward',
|
1087
|
+
stdin_data=patch, cwd=patch_root, tries=1)
|
1088
|
+
for filename, _ in patches:
|
1089
|
+
full_filename = path.abspath(path.join(patch_root, filename))
|
1090
|
+
git('add', full_filename, cwd=path.dirname(full_filename))
|
1091
|
+
except SubprocessFailed as e:
|
1092
|
+
raise PatchFailed(e.message, e.code, e.output)
|
1093
|
+
|
1094
|
+
def apply_rietveld_issue(issue, patchset, root, server, _rev_map, _revision,
|
1095
|
+
email_file, key_file, whitelist=None, blacklist=None):
|
1096
|
+
apply_issue_bin = ('apply_issue.bat' if sys.platform.startswith('win')
|
1097
|
+
else 'apply_issue')
|
1098
|
+
cmd = [apply_issue_bin,
|
1099
|
+
# The patch will be applied on top of this directory.
|
1100
|
+
'--root_dir', root,
|
1101
|
+
# Tell apply_issue how to fetch the patch.
|
1102
|
+
'--issue', issue,
|
1103
|
+
'--server', server,
|
1104
|
+
# Always run apply_issue.py, otherwise it would see update.flag
|
1105
|
+
# and then bail out.
|
1106
|
+
'--force',
|
1107
|
+
# Don't run gclient sync when it sees a DEPS change.
|
1108
|
+
'--ignore_deps',
|
1109
|
+
# TODO(tandrii): remove after http://crbug.com/537417 is resolved.
|
1110
|
+
# Temporary enable verbosity to see if Rietveld requests are actually
|
1111
|
+
# retried.
|
1112
|
+
'-v', '-v', # = logging.DEBUG level.
|
1113
|
+
]
|
1114
|
+
# Use an oauth key file if specified.
|
1115
|
+
if email_file and key_file:
|
1116
|
+
cmd.extend(['--email-file', email_file, '--private-key-file', key_file])
|
1117
|
+
else:
|
1118
|
+
cmd.append('--no-auth')
|
1119
|
+
|
1120
|
+
if patchset:
|
1121
|
+
cmd.extend(['--patchset', patchset])
|
1122
|
+
if whitelist:
|
1123
|
+
for item in whitelist:
|
1124
|
+
cmd.extend(['--whitelist', item])
|
1125
|
+
elif blacklist:
|
1126
|
+
for item in blacklist:
|
1127
|
+
cmd.extend(['--blacklist', item])
|
1128
|
+
|
1129
|
+
# Only try once, since subsequent failures hide the real failure.
|
1130
|
+
try:
|
1131
|
+
call(*cmd, tries=1)
|
1132
|
+
except SubprocessFailed as e:
|
1133
|
+
raise PatchFailed(e.message, e.code, e.output)
|
1134
|
+
|
1135
|
+
def apply_gerrit_ref(gerrit_repo, gerrit_ref, root, gerrit_reset):
|
1136
|
+
gerrit_repo = gerrit_repo or 'origin'
|
1137
|
+
assert gerrit_ref
|
1138
|
+
try:
|
1139
|
+
base_rev = git('rev-parse', 'HEAD', cwd=root).strip()
|
1140
|
+
git('retry', 'fetch', gerrit_repo, gerrit_ref, cwd=root, tries=1)
|
1141
|
+
git('checkout', 'FETCH_HEAD', cwd=root)
|
1142
|
+
if gerrit_reset:
|
1143
|
+
git('reset', '--soft', base_rev, cwd=root)
|
1144
|
+
except SubprocessFailed as e:
|
1145
|
+
raise PatchFailed(e.message, e.code, e.output)
|
1146
|
+
|
1147
|
+
def check_flag(flag_file):
|
1148
|
+
"""Returns True if the flag file is present."""
|
1149
|
+
return os.path.isfile(flag_file)
|
1150
|
+
|
1151
|
+
|
1152
|
+
def delete_flag(flag_file):
|
1153
|
+
"""Remove bot update flag."""
|
1154
|
+
if os.path.isfile(flag_file):
|
1155
|
+
os.remove(flag_file)
|
1156
|
+
|
1157
|
+
|
1158
|
+
def emit_flag(flag_file):
|
1159
|
+
"""Deposit a bot update flag on the system to tell gclient not to run."""
|
1160
|
+
print 'Emitting flag file at %s' % flag_file
|
1161
|
+
with open(flag_file, 'wb') as f:
|
1162
|
+
f.write('Success!')
|
1163
|
+
|
1164
|
+
|
1165
|
+
def get_commit_position_for_git_svn(url, revision):
|
1166
|
+
"""Generates a commit position string for a 'git-svn' URL/revision.
|
1167
|
+
|
1168
|
+
If the 'git-svn' URL maps to a known project, we will construct a commit
|
1169
|
+
position branch value by applying substitution on the SVN URL.
|
1170
|
+
"""
|
1171
|
+
# Identify the base URL so we can strip off trunk/branch name
|
1172
|
+
project_config = branch = None
|
1173
|
+
for _, project_config in GIT_SVN_PROJECT_MAP.iteritems():
|
1174
|
+
if url.startswith(project_config['svn_url']):
|
1175
|
+
branch = url[len(project_config['svn_url']):]
|
1176
|
+
break
|
1177
|
+
|
1178
|
+
if branch:
|
1179
|
+
# Strip any leading slashes
|
1180
|
+
branch = branch.lstrip('/')
|
1181
|
+
|
1182
|
+
# Try and map the branch
|
1183
|
+
for pattern, repl in project_config.get('branch_map', ()):
|
1184
|
+
nbranch, subn = re.subn(pattern, repl, branch, count=1)
|
1185
|
+
if subn:
|
1186
|
+
print 'INFO: Mapped SVN branch to Git branch [%s] => [%s]' % (
|
1187
|
+
branch, nbranch)
|
1188
|
+
branch = nbranch
|
1189
|
+
break
|
1190
|
+
else:
|
1191
|
+
# Use generic 'svn' branch
|
1192
|
+
print 'INFO: Could not resolve project for SVN URL %r' % (url,)
|
1193
|
+
branch = 'svn'
|
1194
|
+
return '%s@{#%s}' % (branch, revision)
|
1195
|
+
|
1196
|
+
|
1197
|
+
def get_commit_position(git_path, revision='HEAD'):
|
1198
|
+
"""Dumps the 'git' log for a specific revision and parses out the commit
|
1199
|
+
position.
|
1200
|
+
|
1201
|
+
If a commit position metadata key is found, its value will be returned.
|
1202
|
+
|
1203
|
+
Otherwise, we will search for a 'git-svn' metadata entry. If one is found,
|
1204
|
+
we will compose a commit position from it, using its SVN revision value as
|
1205
|
+
the revision.
|
1206
|
+
|
1207
|
+
If the 'git-svn' URL maps to a known project, we will construct a commit
|
1208
|
+
position branch value by truncating the URL, mapping 'trunk' to
|
1209
|
+
"refs/heads/master". Otherwise, we will return the generic branch, 'svn'.
|
1210
|
+
"""
|
1211
|
+
git_log = git('log', '--format=%B', '-n1', revision, cwd=git_path)
|
1212
|
+
footer_map = get_commit_message_footer_map(git_log)
|
1213
|
+
|
1214
|
+
# Search for commit position metadata
|
1215
|
+
value = (footer_map.get(COMMIT_POSITION_FOOTER_KEY) or
|
1216
|
+
footer_map.get(COMMIT_ORIGINAL_POSITION_FOOTER_KEY))
|
1217
|
+
if value:
|
1218
|
+
return value
|
1219
|
+
|
1220
|
+
# Compose a commit position from 'git-svn' metadata
|
1221
|
+
value = footer_map.get(GIT_SVN_ID_FOOTER_KEY)
|
1222
|
+
if value:
|
1223
|
+
m = GIT_SVN_ID_RE.match(value)
|
1224
|
+
if not m:
|
1225
|
+
raise ValueError("Invalid 'git-svn' value: [%s]" % (value,))
|
1226
|
+
return get_commit_position_for_git_svn(m.group(1), m.group(2))
|
1227
|
+
return None
|
1228
|
+
|
1229
|
+
|
1230
|
+
def parse_got_revision(gclient_output, got_revision_mapping, use_svn_revs):
|
1231
|
+
"""Translate git gclient revision mapping to build properties.
|
1232
|
+
|
1233
|
+
If use_svn_revs is True, then translate git hashes in the revision mapping
|
1234
|
+
to svn revision numbers.
|
1235
|
+
"""
|
1236
|
+
properties = {}
|
1237
|
+
solutions_output = {
|
1238
|
+
# Make sure path always ends with a single slash.
|
1239
|
+
'%s/' % path.rstrip('/') : solution_output for path, solution_output
|
1240
|
+
in gclient_output['solutions'].iteritems()
|
1241
|
+
}
|
1242
|
+
for dir_name, property_name in got_revision_mapping.iteritems():
|
1243
|
+
# Make sure dir_name always ends with a single slash.
|
1244
|
+
dir_name = '%s/' % dir_name.rstrip('/')
|
1245
|
+
if dir_name not in solutions_output:
|
1246
|
+
continue
|
1247
|
+
solution_output = solutions_output[dir_name]
|
1248
|
+
if solution_output.get('scm') is None:
|
1249
|
+
# This is an ignored DEPS, so the output got_revision should be 'None'.
|
1250
|
+
git_revision = revision = commit_position = None
|
1251
|
+
else:
|
1252
|
+
# Since we are using .DEPS.git, everything had better be git.
|
1253
|
+
assert solution_output.get('scm') == 'git'
|
1254
|
+
git_revision = git('rev-parse', 'HEAD', cwd=dir_name).strip()
|
1255
|
+
if use_svn_revs:
|
1256
|
+
revision = get_svn_rev(git_revision, dir_name)
|
1257
|
+
if not revision:
|
1258
|
+
revision = git_revision
|
1259
|
+
else:
|
1260
|
+
revision = git_revision
|
1261
|
+
commit_position = get_commit_position(dir_name)
|
1262
|
+
|
1263
|
+
properties[property_name] = revision
|
1264
|
+
if revision != git_revision:
|
1265
|
+
properties['%s_git' % property_name] = git_revision
|
1266
|
+
if commit_position:
|
1267
|
+
properties['%s_cp' % property_name] = commit_position
|
1268
|
+
|
1269
|
+
return properties
|
1270
|
+
|
1271
|
+
|
1272
|
+
def emit_json(out_file, did_run, gclient_output=None, **kwargs):
|
1273
|
+
"""Write run information into a JSON file."""
|
1274
|
+
output = {}
|
1275
|
+
output.update(gclient_output if gclient_output else {})
|
1276
|
+
output.update({'did_run': did_run})
|
1277
|
+
output.update(kwargs)
|
1278
|
+
with open(out_file, 'wb') as f:
|
1279
|
+
f.write(json.dumps(output, sort_keys=True))
|
1280
|
+
|
1281
|
+
|
1282
|
+
def ensure_deps_revisions(deps_url_mapping, solutions, revisions):
|
1283
|
+
"""Ensure correct DEPS revisions, ignores solutions."""
|
1284
|
+
for deps_name, deps_data in sorted(deps_url_mapping.items()):
|
1285
|
+
if deps_name.strip('/') in solutions:
|
1286
|
+
# This has already been forced to the correct solution by git_checkout().
|
1287
|
+
continue
|
1288
|
+
revision = get_target_revision(deps_name, deps_data.get('url', None),
|
1289
|
+
revisions)
|
1290
|
+
if not revision:
|
1291
|
+
continue
|
1292
|
+
# TODO(hinoka): Catch SVNRevisionNotFound error maybe?
|
1293
|
+
git('fetch', 'origin', cwd=deps_name)
|
1294
|
+
force_revision(deps_name, revision)
|
1295
|
+
|
1296
|
+
|
1297
|
+
def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only,
|
1298
|
+
patch_root, issue, patchset, patch_url, rietveld_server,
|
1299
|
+
gerrit_repo, gerrit_ref, revision_mapping,
|
1300
|
+
apply_issue_email_file, apply_issue_key_file, buildspec,
|
1301
|
+
gyp_env, shallow, runhooks, refs, git_cache_dir,
|
1302
|
+
gerrit_reset):
|
1303
|
+
# Get a checkout of each solution, without DEPS or hooks.
|
1304
|
+
# Calling git directly because there is no way to run Gclient without
|
1305
|
+
# invoking DEPS.
|
1306
|
+
print 'Fetching Git checkout'
|
1307
|
+
|
1308
|
+
git_ref = git_checkout(solutions, revisions, shallow, refs, git_cache_dir)
|
1309
|
+
|
1310
|
+
patches = None
|
1311
|
+
if patch_url:
|
1312
|
+
patches = get_svn_patch(patch_url)
|
1313
|
+
|
1314
|
+
already_patched = []
|
1315
|
+
patch_root = patch_root or ''
|
1316
|
+
for solution in solutions:
|
1317
|
+
if (patch_root == solution['name'] or
|
1318
|
+
solution['name'].startswith(patch_root + '/')):
|
1319
|
+
relative_root = solution['name'][len(patch_root) + 1:]
|
1320
|
+
target = '/'.join([relative_root, 'DEPS']).lstrip('/')
|
1321
|
+
if patches:
|
1322
|
+
apply_svn_patch(patch_root, patches, whitelist=[target])
|
1323
|
+
already_patched.append(target)
|
1324
|
+
elif issue:
|
1325
|
+
apply_rietveld_issue(issue, patchset, patch_root, rietveld_server,
|
1326
|
+
revision_mapping, git_ref, apply_issue_email_file,
|
1327
|
+
apply_issue_key_file, whitelist=[target])
|
1328
|
+
already_patched.append(target)
|
1329
|
+
|
1330
|
+
if not buildspec:
|
1331
|
+
# Run deps2git if there is a DEPS change after the last .DEPS.git commit.
|
1332
|
+
for solution in solutions:
|
1333
|
+
ensure_deps2git(solution, shallow, git_cache_dir)
|
1334
|
+
|
1335
|
+
# Ensure our build/ directory is set up with the correct .gclient file.
|
1336
|
+
gclient_configure(solutions, target_os, target_os_only, git_cache_dir)
|
1337
|
+
|
1338
|
+
# Let gclient do the DEPS syncing.
|
1339
|
+
# The branch-head refspec is a special case because its possible Chrome
|
1340
|
+
# src, which contains the branch-head refspecs, is DEPSed in.
|
1341
|
+
gclient_output = gclient_sync(buildspec or BRANCH_HEADS_REFSPEC in refs,
|
1342
|
+
shallow)
|
1343
|
+
|
1344
|
+
# Now that gclient_sync has finished, we should revert any .DEPS.git so that
|
1345
|
+
# presubmit doesn't complain about it being modified.
|
1346
|
+
if (not buildspec and
|
1347
|
+
git('ls-files', '.DEPS.git', cwd=first_sln).strip()):
|
1348
|
+
git('checkout', 'HEAD', '--', '.DEPS.git', cwd=first_sln)
|
1349
|
+
|
1350
|
+
if buildspec and runhooks:
|
1351
|
+
# Run gclient runhooks if we're on an official builder.
|
1352
|
+
# TODO(hinoka): Remove this when the official builders run their own
|
1353
|
+
# runhooks step.
|
1354
|
+
gclient_runhooks(gyp_env)
|
1355
|
+
|
1356
|
+
# Finally, ensure that all DEPS are pinned to the correct revision.
|
1357
|
+
dir_names = [sln['name'] for sln in solutions]
|
1358
|
+
ensure_deps_revisions(gclient_output.get('solutions', {}),
|
1359
|
+
dir_names, revisions)
|
1360
|
+
# Apply the rest of the patch here (sans DEPS)
|
1361
|
+
if patches:
|
1362
|
+
apply_svn_patch(patch_root, patches, blacklist=already_patched)
|
1363
|
+
elif issue:
|
1364
|
+
apply_rietveld_issue(issue, patchset, patch_root, rietveld_server,
|
1365
|
+
revision_mapping, git_ref, apply_issue_email_file,
|
1366
|
+
apply_issue_key_file, blacklist=already_patched)
|
1367
|
+
elif gerrit_ref:
|
1368
|
+
apply_gerrit_ref(gerrit_repo, gerrit_ref, patch_root, gerrit_reset)
|
1369
|
+
|
1370
|
+
# Reset the deps_file point in the solutions so that hooks get run properly.
|
1371
|
+
for sln in solutions:
|
1372
|
+
sln['deps_file'] = sln.get('deps_file', 'DEPS').replace('.DEPS.git', 'DEPS')
|
1373
|
+
gclient_configure(solutions, target_os, target_os_only, git_cache_dir)
|
1374
|
+
|
1375
|
+
return gclient_output
|
1376
|
+
|
1377
|
+
|
1378
|
+
def parse_revisions(revisions, root):
|
1379
|
+
"""Turn a list of revision specs into a nice dictionary.
|
1380
|
+
|
1381
|
+
We will always return a dict with {root: something}. By default if root
|
1382
|
+
is unspecified, or if revisions is [], then revision will be assigned 'HEAD'
|
1383
|
+
"""
|
1384
|
+
results = {root.strip('/'): 'HEAD'}
|
1385
|
+
expanded_revisions = []
|
1386
|
+
for revision in revisions:
|
1387
|
+
# Allow rev1,rev2,rev3 format.
|
1388
|
+
# TODO(hinoka): Delete this when webkit switches to recipes.
|
1389
|
+
expanded_revisions.extend(revision.split(','))
|
1390
|
+
for revision in expanded_revisions:
|
1391
|
+
split_revision = revision.split('@')
|
1392
|
+
if len(split_revision) == 1:
|
1393
|
+
# This is just a plain revision, set it as the revision for root.
|
1394
|
+
results[root] = split_revision[0]
|
1395
|
+
elif len(split_revision) == 2:
|
1396
|
+
# This is an alt_root@revision argument.
|
1397
|
+
current_root, current_rev = split_revision
|
1398
|
+
|
1399
|
+
# We want to normalize svn/git urls into .git urls.
|
1400
|
+
parsed_root = urlparse.urlparse(current_root)
|
1401
|
+
if parsed_root.scheme == 'svn':
|
1402
|
+
if parsed_root.path in RECOGNIZED_PATHS:
|
1403
|
+
normalized_root = RECOGNIZED_PATHS[parsed_root.path]
|
1404
|
+
else:
|
1405
|
+
print 'WARNING: SVN path %s not recognized, ignoring' % current_root
|
1406
|
+
continue
|
1407
|
+
elif parsed_root.scheme in ['http', 'https']:
|
1408
|
+
normalized_root = 'https://%s/%s' % (parsed_root.netloc,
|
1409
|
+
parsed_root.path)
|
1410
|
+
if not normalized_root.endswith('.git'):
|
1411
|
+
normalized_root = '%s.git' % normalized_root
|
1412
|
+
elif parsed_root.scheme:
|
1413
|
+
print 'WARNING: Unrecognized scheme %s, ignoring' % parsed_root.scheme
|
1414
|
+
continue
|
1415
|
+
else:
|
1416
|
+
# This is probably a local path.
|
1417
|
+
normalized_root = current_root.strip('/')
|
1418
|
+
|
1419
|
+
results[normalized_root] = current_rev
|
1420
|
+
else:
|
1421
|
+
print ('WARNING: %r is not recognized as a valid revision specification,'
|
1422
|
+
'skipping' % revision)
|
1423
|
+
return results
|
1424
|
+
|
1425
|
+
|
1426
|
+
def parse_args():
|
1427
|
+
parse = optparse.OptionParser()
|
1428
|
+
|
1429
|
+
parse.add_option('--issue', help='Issue number to patch from.')
|
1430
|
+
parse.add_option('--patchset',
|
1431
|
+
help='Patchset from issue to patch from, if applicable.')
|
1432
|
+
parse.add_option('--apply_issue_email_file',
|
1433
|
+
help='--email-file option passthrough for apply_patch.py.')
|
1434
|
+
parse.add_option('--apply_issue_key_file',
|
1435
|
+
help='--private-key-file option passthrough for '
|
1436
|
+
'apply_patch.py.')
|
1437
|
+
parse.add_option('--patch_url', help='Optional URL to SVN patch.')
|
1438
|
+
parse.add_option('--root', dest='patch_root',
|
1439
|
+
help='DEPRECATED: Use --patch_root.')
|
1440
|
+
parse.add_option('--patch_root', help='Directory to patch on top of.')
|
1441
|
+
parse.add_option('--rietveld_server',
|
1442
|
+
default='codereview.chromium.org',
|
1443
|
+
help='Rietveld server.')
|
1444
|
+
parse.add_option('--gerrit_repo',
|
1445
|
+
help='Gerrit repository to pull the ref from.')
|
1446
|
+
parse.add_option('--gerrit_ref', help='Gerrit ref to apply.')
|
1447
|
+
parse.add_option('--gerrit_no_reset', action='store_true',
|
1448
|
+
help='Bypass calling reset after applying a gerrit ref.')
|
1449
|
+
parse.add_option('--specs', help='Gcilent spec.')
|
1450
|
+
parse.add_option('--master', help='Master name.')
|
1451
|
+
parse.add_option('-f', '--force', action='store_true',
|
1452
|
+
help='Bypass check to see if we want to be run. '
|
1453
|
+
'Should ONLY be used locally or by smart recipes.')
|
1454
|
+
parse.add_option('--revision_mapping',
|
1455
|
+
help='{"path/to/repo/": "property_name"}')
|
1456
|
+
parse.add_option('--revision_mapping_file',
|
1457
|
+
help=('Same as revision_mapping, except its a path to a json'
|
1458
|
+
' file containing that format.'))
|
1459
|
+
parse.add_option('--revision', action='append', default=[],
|
1460
|
+
help='Revision to check out. Can be an SVN revision number, '
|
1461
|
+
'git hash, or any form of git ref. Can prepend '
|
1462
|
+
'root@<rev> to specify which repository, where root '
|
1463
|
+
'is either a filesystem path, git https url, or '
|
1464
|
+
'svn url. To specify Tip of Tree, set rev to HEAD.'
|
1465
|
+
'To specify a git branch and an SVN rev, <rev> can be '
|
1466
|
+
'set to <branch>:<revision>.')
|
1467
|
+
parse.add_option('--output_manifest', action='store_true',
|
1468
|
+
help=('Add manifest json to the json output.'))
|
1469
|
+
parse.add_option('--slave_name', default=socket.getfqdn().split('.')[0],
|
1470
|
+
help='Hostname of the current machine, '
|
1471
|
+
'used for determining whether or not to activate.')
|
1472
|
+
parse.add_option('--builder_name', help='Name of the builder, '
|
1473
|
+
'used for determining whether or not to activate.')
|
1474
|
+
parse.add_option('--build_dir', default=os.getcwd())
|
1475
|
+
parse.add_option('--flag_file', default=path.join(os.getcwd(),
|
1476
|
+
'update.flag'))
|
1477
|
+
parse.add_option('--shallow', action='store_true',
|
1478
|
+
help='Use shallow clones for cache repositories.')
|
1479
|
+
parse.add_option('--gyp_env', action='append', default=[],
|
1480
|
+
help='Environment variables to pass into gclient runhooks.')
|
1481
|
+
parse.add_option('--clobber', action='store_true',
|
1482
|
+
help='Delete checkout first, always')
|
1483
|
+
parse.add_option('--bot_update_clobber', action='store_true', dest='clobber',
|
1484
|
+
help='(synonym for --clobber)')
|
1485
|
+
parse.add_option('-o', '--output_json',
|
1486
|
+
help='Output JSON information into a specified file')
|
1487
|
+
parse.add_option('--no_shallow', action='store_true',
|
1488
|
+
help='Bypass disk detection and never shallow clone. '
|
1489
|
+
'Does not override the --shallow flag')
|
1490
|
+
parse.add_option('--no_runhooks', action='store_true',
|
1491
|
+
help='Do not run hooks on official builder.')
|
1492
|
+
parse.add_option('--refs', action='append',
|
1493
|
+
help='Also fetch this refspec for the main solution(s). '
|
1494
|
+
'Eg. +refs/branch-heads/*')
|
1495
|
+
parse.add_option('--with_branch_heads', action='store_true',
|
1496
|
+
help='Always pass --with_branch_heads to gclient. This '
|
1497
|
+
'does the same thing as --refs +refs/branch-heads/*')
|
1498
|
+
parse.add_option('--git-cache-dir', default=path.join(SLAVE_DIR, 'cache_dir'),
|
1499
|
+
help='Path to git cache directory.')
|
1500
|
+
|
1501
|
+
|
1502
|
+
options, args = parse.parse_args()
|
1503
|
+
|
1504
|
+
if not options.refs:
|
1505
|
+
options.refs = []
|
1506
|
+
|
1507
|
+
if options.with_branch_heads:
|
1508
|
+
options.refs.append(BRANCH_HEADS_REFSPEC)
|
1509
|
+
del options.with_branch_heads
|
1510
|
+
|
1511
|
+
try:
|
1512
|
+
if options.revision_mapping_file:
|
1513
|
+
if options.revision_mapping:
|
1514
|
+
print ('WARNING: Ignoring --revision_mapping: --revision_mapping_file '
|
1515
|
+
'was set at the same time as --revision_mapping?')
|
1516
|
+
with open(options.revision_mapping_file, 'r') as f:
|
1517
|
+
options.revision_mapping = json.load(f)
|
1518
|
+
elif options.revision_mapping:
|
1519
|
+
options.revision_mapping = json.loads(options.revision_mapping)
|
1520
|
+
except Exception as e:
|
1521
|
+
print (
|
1522
|
+
'WARNING: Caught execption while parsing revision_mapping*: %s'
|
1523
|
+
% (str(e),)
|
1524
|
+
)
|
1525
|
+
|
1526
|
+
# Because we print CACHE_DIR out into a .gclient file, and then later run
|
1527
|
+
# eval() on it, backslashes need to be escaped, otherwise "E:\b\build" gets
|
1528
|
+
# parsed as "E:[\x08][\x08]uild".
|
1529
|
+
if sys.platform.startswith('win'):
|
1530
|
+
options.git_cache_dir = options.git_cache_dir.replace('\\', '\\\\')
|
1531
|
+
|
1532
|
+
return options, args
|
1533
|
+
|
1534
|
+
|
1535
|
+
def prepare(options, git_slns, active):
|
1536
|
+
"""Prepares the target folder before we checkout."""
|
1537
|
+
dir_names = [sln.get('name') for sln in git_slns if 'name' in sln]
|
1538
|
+
# If we're active now, but the flag file doesn't exist (we weren't active
|
1539
|
+
# last run) or vice versa, blow away all checkouts.
|
1540
|
+
if bool(active) != bool(check_flag(options.flag_file)):
|
1541
|
+
ensure_no_checkout(dir_names, '*')
|
1542
|
+
if options.output_json:
|
1543
|
+
# Make sure we tell recipes that we didn't run if the script exits here.
|
1544
|
+
emit_json(options.output_json, did_run=active)
|
1545
|
+
if active:
|
1546
|
+
if options.clobber:
|
1547
|
+
ensure_no_checkout(dir_names, '*')
|
1548
|
+
else:
|
1549
|
+
ensure_no_checkout(dir_names, '.svn')
|
1550
|
+
emit_flag(options.flag_file)
|
1551
|
+
else:
|
1552
|
+
delete_flag(options.flag_file)
|
1553
|
+
raise Inactive # This is caught in main() and we exit cleanly.
|
1554
|
+
|
1555
|
+
# Do a shallow checkout if the disk is less than 100GB.
|
1556
|
+
total_disk_space, free_disk_space = get_total_disk_space()
|
1557
|
+
total_disk_space_gb = int(total_disk_space / (1024 * 1024 * 1024))
|
1558
|
+
used_disk_space_gb = int((total_disk_space - free_disk_space)
|
1559
|
+
/ (1024 * 1024 * 1024))
|
1560
|
+
percent_used = int(used_disk_space_gb * 100 / total_disk_space_gb)
|
1561
|
+
step_text = '[%dGB/%dGB used (%d%%)]' % (used_disk_space_gb,
|
1562
|
+
total_disk_space_gb,
|
1563
|
+
percent_used)
|
1564
|
+
if not options.output_json:
|
1565
|
+
print '@@@STEP_TEXT@%s@@@' % step_text
|
1566
|
+
if not options.shallow:
|
1567
|
+
options.shallow = (total_disk_space < SHALLOW_CLONE_THRESHOLD
|
1568
|
+
and not options.no_shallow)
|
1569
|
+
|
1570
|
+
# The first solution is where the primary DEPS file resides.
|
1571
|
+
first_sln = dir_names[0]
|
1572
|
+
|
1573
|
+
# Split all the revision specifications into a nice dict.
|
1574
|
+
print 'Revisions: %s' % options.revision
|
1575
|
+
revisions = parse_revisions(options.revision, first_sln)
|
1576
|
+
print 'Fetching Git checkout at %s@%s' % (first_sln, revisions[first_sln])
|
1577
|
+
return revisions, step_text
|
1578
|
+
|
1579
|
+
|
1580
|
+
def checkout(options, git_slns, specs, buildspec, master,
|
1581
|
+
svn_root, revisions, step_text):
|
1582
|
+
first_sln = git_slns[0]['name']
|
1583
|
+
dir_names = [sln.get('name') for sln in git_slns if 'name' in sln]
|
1584
|
+
try:
|
1585
|
+
# Outer try is for catching patch failures and exiting gracefully.
|
1586
|
+
# Inner try is for catching gclient failures and retrying gracefully.
|
1587
|
+
try:
|
1588
|
+
checkout_parameters = dict(
|
1589
|
+
# First, pass in the base of what we want to check out.
|
1590
|
+
solutions=git_slns,
|
1591
|
+
revisions=revisions,
|
1592
|
+
first_sln=first_sln,
|
1593
|
+
|
1594
|
+
# Also, target os variables for gclient.
|
1595
|
+
target_os=specs.get('target_os', []),
|
1596
|
+
target_os_only=specs.get('target_os_only', False),
|
1597
|
+
|
1598
|
+
# Then, pass in information about how to patch.
|
1599
|
+
patch_root=options.patch_root,
|
1600
|
+
issue=options.issue,
|
1601
|
+
patchset=options.patchset,
|
1602
|
+
patch_url=options.patch_url,
|
1603
|
+
rietveld_server=options.rietveld_server,
|
1604
|
+
gerrit_repo=options.gerrit_repo,
|
1605
|
+
gerrit_ref=options.gerrit_ref,
|
1606
|
+
revision_mapping=options.revision_mapping,
|
1607
|
+
apply_issue_email_file=options.apply_issue_email_file,
|
1608
|
+
apply_issue_key_file=options.apply_issue_key_file,
|
1609
|
+
|
1610
|
+
# For official builders.
|
1611
|
+
buildspec=buildspec,
|
1612
|
+
gyp_env=options.gyp_env,
|
1613
|
+
runhooks=not options.no_runhooks,
|
1614
|
+
|
1615
|
+
# Finally, extra configurations such as shallowness of the clone.
|
1616
|
+
shallow=options.shallow,
|
1617
|
+
refs=options.refs,
|
1618
|
+
git_cache_dir=options.git_cache_dir,
|
1619
|
+
gerrit_reset=not options.gerrit_no_reset)
|
1620
|
+
gclient_output = ensure_checkout(**checkout_parameters)
|
1621
|
+
except GclientSyncFailed:
|
1622
|
+
print 'We failed gclient sync, lets delete the checkout and retry.'
|
1623
|
+
ensure_no_checkout(dir_names, '*')
|
1624
|
+
gclient_output = ensure_checkout(**checkout_parameters)
|
1625
|
+
except PatchFailed as e:
|
1626
|
+
if options.output_json:
|
1627
|
+
# Tell recipes information such as root, got_revision, etc.
|
1628
|
+
emit_json(options.output_json,
|
1629
|
+
did_run=True,
|
1630
|
+
root=first_sln,
|
1631
|
+
log_lines=[('patch error', e.output),],
|
1632
|
+
patch_apply_return_code=e.code,
|
1633
|
+
patch_root=options.patch_root,
|
1634
|
+
patch_failure=True,
|
1635
|
+
step_text='%s PATCH FAILED' % step_text,
|
1636
|
+
fixed_revisions=revisions)
|
1637
|
+
else:
|
1638
|
+
# If we're not on recipes, tell annotator about our got_revisions.
|
1639
|
+
emit_log_lines('patch error', e.output)
|
1640
|
+
print '@@@STEP_TEXT@%s PATCH FAILED@@@' % step_text
|
1641
|
+
raise
|
1642
|
+
|
1643
|
+
# Revision is an svn revision, unless it's a git master.
|
1644
|
+
use_svn_rev = master not in GIT_MASTERS
|
1645
|
+
|
1646
|
+
# Take care of got_revisions outputs.
|
1647
|
+
revision_mapping = dict(GOT_REVISION_MAPPINGS.get(svn_root, {}))
|
1648
|
+
if options.revision_mapping:
|
1649
|
+
revision_mapping.update(options.revision_mapping)
|
1650
|
+
|
1651
|
+
# If the repo is not in the default GOT_REVISION_MAPPINGS and no
|
1652
|
+
# revision_mapping were specified on the command line then
|
1653
|
+
# default to setting 'got_revision' based on the first solution.
|
1654
|
+
if not revision_mapping:
|
1655
|
+
revision_mapping[first_sln] = 'got_revision'
|
1656
|
+
|
1657
|
+
got_revisions = parse_got_revision(gclient_output, revision_mapping,
|
1658
|
+
use_svn_rev)
|
1659
|
+
|
1660
|
+
if not got_revisions:
|
1661
|
+
# TODO(hinoka): We should probably bail out here, but in the interest
|
1662
|
+
# of giving mis-configured bots some time to get fixed use a dummy
|
1663
|
+
# revision here.
|
1664
|
+
got_revisions = { 'got_revision': 'BOT_UPDATE_NO_REV_FOUND' }
|
1665
|
+
#raise Exception('No got_revision(s) found in gclient output')
|
1666
|
+
|
1667
|
+
if options.output_json:
|
1668
|
+
manifest = create_manifest() if options.output_manifest else None
|
1669
|
+
# Tell recipes information such as root, got_revision, etc.
|
1670
|
+
emit_json(options.output_json,
|
1671
|
+
did_run=True,
|
1672
|
+
root=first_sln,
|
1673
|
+
patch_root=options.patch_root,
|
1674
|
+
step_text=step_text,
|
1675
|
+
fixed_revisions=revisions,
|
1676
|
+
properties=got_revisions,
|
1677
|
+
manifest=manifest)
|
1678
|
+
else:
|
1679
|
+
# If we're not on recipes, tell annotator about our got_revisions.
|
1680
|
+
emit_properties(got_revisions)
|
1681
|
+
|
1682
|
+
|
1683
|
+
def print_help_text(force, output_json, active, master, builder, slave):
|
1684
|
+
"""Print helpful messages to tell devs whats going on."""
|
1685
|
+
if force and output_json:
|
1686
|
+
recipe_force = 'Forced on by recipes'
|
1687
|
+
elif active and output_json:
|
1688
|
+
recipe_force = 'Off by recipes, but forced on by bot update'
|
1689
|
+
elif not active and output_json:
|
1690
|
+
recipe_force = 'Forced off by recipes'
|
1691
|
+
else:
|
1692
|
+
recipe_force = 'N/A. Was not called by recipes'
|
1693
|
+
|
1694
|
+
print BOT_UPDATE_MESSAGE % {
|
1695
|
+
'master': master or 'Not specified',
|
1696
|
+
'builder': builder or 'Not specified',
|
1697
|
+
'slave': slave or 'Not specified',
|
1698
|
+
'recipe': recipe_force,
|
1699
|
+
'CURRENT_DIR': CURRENT_DIR,
|
1700
|
+
'BUILDER_DIR': BUILDER_DIR,
|
1701
|
+
'SLAVE_DIR': SLAVE_DIR,
|
1702
|
+
'THIS_DIR': THIS_DIR,
|
1703
|
+
'SCRIPTS_DIR': SCRIPTS_DIR,
|
1704
|
+
'BUILD_DIR': BUILD_DIR,
|
1705
|
+
'ROOT_DIR': ROOT_DIR,
|
1706
|
+
'DEPOT_TOOLS_DIR': DEPOT_TOOLS_DIR,
|
1707
|
+
},
|
1708
|
+
print ACTIVATED_MESSAGE if active else NOT_ACTIVATED_MESSAGE
|
1709
|
+
|
1710
|
+
|
1711
|
+
def main():
|
1712
|
+
# Get inputs.
|
1713
|
+
options, _ = parse_args()
|
1714
|
+
builder = options.builder_name
|
1715
|
+
slave = options.slave_name
|
1716
|
+
master = options.master
|
1717
|
+
|
1718
|
+
# Check if this script should activate or not.
|
1719
|
+
active = check_valid_host(master, builder, slave) or options.force or False
|
1720
|
+
|
1721
|
+
# Print a helpful message to tell developers whats going on with this step.
|
1722
|
+
print_help_text(
|
1723
|
+
options.force, options.output_json, active, master, builder, slave)
|
1724
|
+
|
1725
|
+
# Parse, munipulate, and print the gclient solutions.
|
1726
|
+
specs = {}
|
1727
|
+
exec(options.specs, specs)
|
1728
|
+
svn_solutions = specs.get('solutions', [])
|
1729
|
+
git_slns, svn_root, buildspec = solutions_to_git(svn_solutions)
|
1730
|
+
options.revision = maybe_ignore_revision(options.revision, buildspec)
|
1731
|
+
|
1732
|
+
solutions_printer(git_slns)
|
1733
|
+
|
1734
|
+
try:
|
1735
|
+
# Dun dun dun, the main part of bot_update.
|
1736
|
+
revisions, step_text = prepare(options, git_slns, active)
|
1737
|
+
checkout(options, git_slns, specs, buildspec, master, svn_root, revisions,
|
1738
|
+
step_text)
|
1739
|
+
|
1740
|
+
except Inactive:
|
1741
|
+
# Not active, should count as passing.
|
1742
|
+
pass
|
1743
|
+
except PatchFailed as e:
|
1744
|
+
emit_flag(options.flag_file)
|
1745
|
+
# Return a specific non-zero exit code for patch failure (because it is
|
1746
|
+
# a failure), but make it different than other failures to distinguish
|
1747
|
+
# between infra failures (independent from patch author), and patch
|
1748
|
+
# failures (that patch author can fix). However, PatchFailure due to
|
1749
|
+
# download patch failure is still an infra problem.
|
1750
|
+
if e.code == 3:
|
1751
|
+
# Patch download problem.
|
1752
|
+
return 87
|
1753
|
+
# Genuine patch problem.
|
1754
|
+
return 88
|
1755
|
+
except Exception:
|
1756
|
+
# Unexpected failure.
|
1757
|
+
emit_flag(options.flag_file)
|
1758
|
+
raise
|
1759
|
+
else:
|
1760
|
+
emit_flag(options.flag_file)
|
1761
|
+
|
1762
|
+
|
1763
|
+
if __name__ == '__main__':
|
1764
|
+
sys.exit(main())
|