libv8 4.5.95.5 → 5.0.71.48.0beta2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (332) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +45 -19
  3. data/CHANGELOG.md +14 -0
  4. data/README.md +30 -15
  5. data/Rakefile +7 -6
  6. data/ext/libv8/arch.rb +5 -4
  7. data/ext/libv8/builder.rb +25 -19
  8. data/ext/libv8/compiler.rb +6 -33
  9. data/ext/libv8/location.rb +7 -8
  10. data/lib/libv8/version.rb +1 -1
  11. data/libv8.gemspec +1 -1
  12. data/patches/build-standalone-static-library.patch +14 -0
  13. data/patches/disable-building-tests.patch +48 -10
  14. data/patches/fPIC-for-static.patch +3 -3
  15. data/release/x86-linux/Vagrantfile +8 -4
  16. data/release/x86_64-freebsd10/Vagrantfile +86 -0
  17. data/release/x86_64-linux/Vagrantfile +8 -4
  18. data/spec/compiler_spec.rb +5 -29
  19. data/spec/support/compiler_helpers.rb +2 -4
  20. data/vendor/depot_tools/.gitignore +15 -3
  21. data/vendor/depot_tools/OWNERS +2 -2
  22. data/vendor/depot_tools/PRESUBMIT.py +4 -2
  23. data/vendor/depot_tools/WATCHLISTS +6 -0
  24. data/vendor/depot_tools/apply_issue.py +70 -38
  25. data/vendor/depot_tools/bootstrap/win/README.md +66 -0
  26. data/vendor/depot_tools/bootstrap/win/git-bash.template.sh +12 -0
  27. data/vendor/depot_tools/bootstrap/win/git.template.bat +5 -0
  28. data/vendor/depot_tools/bootstrap/win/profile.d.python.sh +20 -0
  29. data/vendor/depot_tools/bootstrap/win/win_tools.bat +96 -45
  30. data/vendor/depot_tools/breakpad.py +6 -141
  31. data/vendor/depot_tools/buildbucket.py +45 -31
  32. data/vendor/depot_tools/cbuildbot +1 -0
  33. data/vendor/depot_tools/checkout.py +2 -1
  34. data/vendor/depot_tools/chrome_set_ver +1 -0
  35. data/vendor/depot_tools/cit +8 -0
  36. data/vendor/depot_tools/cit.bat +11 -0
  37. data/vendor/depot_tools/cit.py +120 -0
  38. data/vendor/depot_tools/codereview.settings +0 -2
  39. data/vendor/depot_tools/commit_queue +1 -5
  40. data/vendor/depot_tools/commit_queue.bat +1 -4
  41. data/vendor/depot_tools/commit_queue.py +78 -29
  42. data/vendor/depot_tools/cpplint.py +22 -14
  43. data/vendor/depot_tools/cros +1 -0
  44. data/vendor/depot_tools/cros_sdk +1 -0
  45. data/vendor/depot_tools/depot-tools-auth.py +3 -3
  46. data/vendor/depot_tools/download_from_google_storage.py +101 -21
  47. data/vendor/depot_tools/drover.py +2 -3
  48. data/vendor/depot_tools/fetch.py +31 -27
  49. data/vendor/depot_tools/{recipes → fetch_configs}/android.py +4 -4
  50. data/vendor/depot_tools/fetch_configs/breakpad.py +45 -0
  51. data/vendor/depot_tools/{recipes → fetch_configs}/chromium.py +3 -3
  52. data/vendor/depot_tools/{recipes/recipe_util.py → fetch_configs/config_util.py} +3 -3
  53. data/vendor/depot_tools/fetch_configs/crashpad.py +41 -0
  54. data/vendor/depot_tools/{recipes → fetch_configs}/dart.py +3 -3
  55. data/vendor/depot_tools/{recipes/pdfium.py → fetch_configs/dartino.py} +14 -13
  56. data/vendor/depot_tools/{recipes → fetch_configs}/dartium.py +3 -3
  57. data/vendor/depot_tools/{recipes → fetch_configs}/depot_tools.py +3 -3
  58. data/vendor/depot_tools/fetch_configs/gyp.py +42 -0
  59. data/vendor/depot_tools/{recipes → fetch_configs}/infra.py +3 -3
  60. data/vendor/depot_tools/{recipes → fetch_configs}/infra_internal.py +3 -3
  61. data/vendor/depot_tools/{recipes → fetch_configs}/ios.py +4 -4
  62. data/vendor/depot_tools/{recipes → fetch_configs}/mojo.py +3 -3
  63. data/vendor/depot_tools/{recipes → fetch_configs}/nacl.py +3 -3
  64. data/vendor/depot_tools/{recipes → fetch_configs}/naclports.py +3 -3
  65. data/vendor/depot_tools/fetch_configs/pdfium.py +40 -0
  66. data/vendor/depot_tools/{recipes → fetch_configs}/skia.py +3 -3
  67. data/vendor/depot_tools/{recipes → fetch_configs}/skia_buildbot.py +3 -3
  68. data/vendor/depot_tools/fetch_configs/syzygy.py +41 -0
  69. data/vendor/depot_tools/{recipes → fetch_configs}/v8.py +3 -3
  70. data/vendor/depot_tools/{recipes → fetch_configs}/webrtc.py +3 -3
  71. data/vendor/depot_tools/{recipes → fetch_configs}/webrtc_android.py +4 -4
  72. data/vendor/depot_tools/{recipes → fetch_configs}/webrtc_ios.py +4 -4
  73. data/vendor/depot_tools/fix_encoding.py +6 -6
  74. data/vendor/depot_tools/gcl.py +11 -21
  75. data/vendor/depot_tools/gclient +10 -0
  76. data/vendor/depot_tools/gclient-new-workdir.py +7 -38
  77. data/vendor/depot_tools/gclient.bat +2 -2
  78. data/vendor/depot_tools/gclient.py +85 -65
  79. data/vendor/depot_tools/gclient_scm.py +83 -10
  80. data/vendor/depot_tools/gclient_utils.py +5 -1
  81. data/vendor/depot_tools/gerrit_util.py +243 -26
  82. data/vendor/depot_tools/git-auto-svn +1 -1
  83. data/vendor/depot_tools/git-cache +1 -1
  84. data/vendor/depot_tools/git-cherry-pick-upload +1 -1
  85. data/vendor/depot_tools/git-cl +1 -1
  86. data/vendor/depot_tools/git-drover +6 -0
  87. data/vendor/depot_tools/git-find-releases +6 -0
  88. data/vendor/depot_tools/git-footers +1 -1
  89. data/vendor/depot_tools/git-freeze +1 -1
  90. data/vendor/depot_tools/git-gs +1 -1
  91. data/vendor/depot_tools/git-hyper-blame +6 -0
  92. data/vendor/depot_tools/git-map +1 -1
  93. data/vendor/depot_tools/git-map-branches +1 -1
  94. data/vendor/depot_tools/git-mark-merge-base +1 -1
  95. data/vendor/depot_tools/git-nav-downstream +1 -1
  96. data/vendor/depot_tools/git-new-branch +1 -1
  97. data/vendor/depot_tools/git-number +1 -1
  98. data/vendor/depot_tools/git-rebase-update +1 -1
  99. data/vendor/depot_tools/git-rename-branch +1 -1
  100. data/vendor/depot_tools/git-reparent-branch +1 -1
  101. data/vendor/depot_tools/git-retry +1 -1
  102. data/vendor/depot_tools/git-squash-branch +1 -1
  103. data/vendor/depot_tools/git-thaw +1 -1
  104. data/vendor/depot_tools/git-try +1 -1
  105. data/vendor/depot_tools/git-upstream-diff +1 -1
  106. data/vendor/depot_tools/git_auto_svn.py +24 -6
  107. data/vendor/depot_tools/git_cache.py +74 -27
  108. data/vendor/depot_tools/git_cl.py +2118 -747
  109. data/vendor/depot_tools/git_common.py +100 -6
  110. data/vendor/depot_tools/git_dates.py +62 -0
  111. data/vendor/depot_tools/git_drover.py +424 -0
  112. data/vendor/depot_tools/git_find_releases.py +65 -0
  113. data/vendor/depot_tools/git_footers.py +42 -0
  114. data/vendor/depot_tools/git_hyper_blame.py +391 -0
  115. data/vendor/depot_tools/git_map_branches.py +8 -6
  116. data/vendor/depot_tools/git_new_branch.py +6 -1
  117. data/vendor/depot_tools/git_rebase_update.py +56 -16
  118. data/vendor/depot_tools/git_reparent_branch.py +13 -0
  119. data/vendor/depot_tools/git_try.py +0 -2
  120. data/vendor/depot_tools/gsutil.py +51 -20
  121. data/vendor/depot_tools/infra/config/OWNERS +3 -1
  122. data/vendor/depot_tools/infra/config/cq.cfg +7 -3
  123. data/vendor/depot_tools/infra/config/recipes.cfg +9 -0
  124. data/vendor/depot_tools/luci_hacks/README.md +35 -0
  125. data/vendor/depot_tools/{bootstrap/virtualenv/tests → luci_hacks}/__init__.py +0 -0
  126. data/vendor/depot_tools/luci_hacks/luci_recipe_run.isolate +12 -0
  127. data/vendor/depot_tools/luci_hacks/luci_recipe_run.py +81 -0
  128. data/vendor/depot_tools/luci_hacks/trigger_luci_job.py +128 -0
  129. data/vendor/depot_tools/man/html/depot_tools.html +9 -1
  130. data/vendor/depot_tools/man/html/depot_tools_tutorial.html +4 -4
  131. data/vendor/depot_tools/man/html/git-drover.html +191 -35
  132. data/vendor/depot_tools/man/html/git-hyper-blame.html +878 -0
  133. data/vendor/depot_tools/man/html/git-rebase-update.html +9 -4
  134. data/vendor/depot_tools/man/man1/git-drover.1 +189 -36
  135. data/vendor/depot_tools/man/man1/git-hyper-blame.1 +128 -0
  136. data/vendor/depot_tools/man/man1/git-rebase-update.1 +8 -6
  137. data/vendor/depot_tools/man/man7/depot_tools.7 +9 -4
  138. data/vendor/depot_tools/man/src/_git-hyper-blame_desc.helper.txt +1 -0
  139. data/vendor/depot_tools/man/src/common_demo_functions.sh +5 -0
  140. data/vendor/depot_tools/man/src/depot_tools_tutorial.txt +1 -1
  141. data/vendor/depot_tools/man/src/git-drover.demo.1.sh +11 -16
  142. data/vendor/depot_tools/man/src/git-drover.demo.3.sh +27 -0
  143. data/vendor/depot_tools/man/src/git-drover.demo.4.sh +39 -0
  144. data/vendor/depot_tools/man/src/git-drover.txt +49 -3
  145. data/vendor/depot_tools/man/src/git-hyper-blame.demo.1.sh +3 -0
  146. data/vendor/depot_tools/man/src/git-hyper-blame.demo.2.sh +4 -0
  147. data/vendor/depot_tools/man/src/git-hyper-blame.demo.common.sh +57 -0
  148. data/vendor/depot_tools/man/src/git-hyper-blame.txt +85 -0
  149. data/vendor/depot_tools/man/src/git-rebase-update.txt +5 -1
  150. data/vendor/depot_tools/my_activity.py +6 -21
  151. data/vendor/depot_tools/ninja +2 -2
  152. data/vendor/depot_tools/ninja-linux32 +0 -0
  153. data/vendor/depot_tools/ninja-linux64 +0 -0
  154. data/vendor/depot_tools/ninja-mac +0 -0
  155. data/vendor/depot_tools/ninja.exe +0 -0
  156. data/vendor/depot_tools/presubmit_canned_checks.py +83 -69
  157. data/vendor/depot_tools/presubmit_support.py +126 -42
  158. data/vendor/depot_tools/pylint.py +5 -1
  159. data/vendor/depot_tools/python_runner.sh +55 -0
  160. data/vendor/depot_tools/recipe_modules/bot_update/__init__.py +32 -0
  161. data/vendor/depot_tools/recipe_modules/bot_update/api.py +283 -0
  162. data/vendor/depot_tools/recipe_modules/bot_update/example.expected/basic.json +56 -0
  163. data/vendor/depot_tools/recipe_modules/bot_update/example.expected/basic_output_manifest.json +63 -0
  164. data/vendor/depot_tools/recipe_modules/bot_update/example.expected/basic_with_branch_heads.json +57 -0
  165. data/vendor/depot_tools/recipe_modules/bot_update/example.expected/clobber.json +44 -0
  166. data/vendor/depot_tools/recipe_modules/bot_update/example.expected/forced.json +57 -0
  167. data/vendor/depot_tools/recipe_modules/bot_update/example.expected/gerrit_no_reset.json +44 -0
  168. data/vendor/depot_tools/recipe_modules/bot_update/example.expected/no_shallow.json +44 -0
  169. data/vendor/depot_tools/recipe_modules/bot_update/example.expected/off.json +43 -0
  170. data/vendor/depot_tools/recipe_modules/bot_update/example.expected/reset_root_solution_revision.json +43 -0
  171. data/vendor/depot_tools/recipe_modules/bot_update/example.expected/svn_mode.json +59 -0
  172. data/vendor/depot_tools/recipe_modules/bot_update/example.expected/trychange.json +58 -0
  173. data/vendor/depot_tools/recipe_modules/bot_update/example.expected/trychange_oauth2.json +60 -0
  174. data/vendor/depot_tools/recipe_modules/bot_update/example.expected/tryjob.json +58 -0
  175. data/vendor/depot_tools/recipe_modules/bot_update/example.expected/tryjob_fail.json +60 -0
  176. data/vendor/depot_tools/recipe_modules/bot_update/example.expected/tryjob_fail_patch.json +81 -0
  177. data/vendor/depot_tools/recipe_modules/bot_update/example.expected/tryjob_fail_patch_download.json +81 -0
  178. data/vendor/depot_tools/recipe_modules/bot_update/example.expected/tryjob_gerrit_angle.json +49 -0
  179. data/vendor/depot_tools/recipe_modules/bot_update/example.expected/tryjob_v8.json +61 -0
  180. data/vendor/depot_tools/recipe_modules/bot_update/example.expected/tryjob_v8_head_by_default.json +51 -0
  181. data/vendor/depot_tools/recipe_modules/bot_update/example.py +172 -0
  182. data/vendor/depot_tools/{bootstrap/virtualenv/virtualenv_support → recipe_modules/bot_update/resources}/__init__.py +0 -0
  183. data/vendor/depot_tools/recipe_modules/bot_update/resources/bot_update.py +1764 -0
  184. data/vendor/depot_tools/recipe_modules/bot_update/test_api.py +86 -0
  185. data/vendor/depot_tools/recipe_modules/depot_tools/__init__.py +3 -0
  186. data/vendor/depot_tools/recipe_modules/depot_tools/api.py +27 -0
  187. data/vendor/depot_tools/recipe_modules/gclient/__init__.py +10 -0
  188. data/vendor/depot_tools/recipe_modules/gclient/api.py +378 -0
  189. data/vendor/depot_tools/recipe_modules/gclient/config.py +671 -0
  190. data/vendor/depot_tools/recipe_modules/gclient/example.expected/basic.json +172 -0
  191. data/vendor/depot_tools/recipe_modules/gclient/example.expected/revision.json +174 -0
  192. data/vendor/depot_tools/recipe_modules/gclient/example.expected/tryserver.json +185 -0
  193. data/vendor/depot_tools/recipe_modules/gclient/example.py +100 -0
  194. data/vendor/depot_tools/recipe_modules/gclient/test_api.py +37 -0
  195. data/vendor/depot_tools/recipe_modules/git/__init__.py +9 -0
  196. data/vendor/depot_tools/recipe_modules/git/api.py +377 -0
  197. data/vendor/depot_tools/recipe_modules/git/example.expected/basic.json +177 -0
  198. data/vendor/depot_tools/recipe_modules/git/example.expected/basic_branch.json +177 -0
  199. data/vendor/depot_tools/recipe_modules/git/example.expected/basic_file_name.json +179 -0
  200. data/vendor/depot_tools/recipe_modules/git/example.expected/basic_hash.json +176 -0
  201. data/vendor/depot_tools/recipe_modules/git/example.expected/basic_ref.json +177 -0
  202. data/vendor/depot_tools/recipe_modules/git/example.expected/basic_submodule_update_force.json +178 -0
  203. data/vendor/depot_tools/recipe_modules/git/example.expected/can_fail_build.json +153 -0
  204. data/vendor/depot_tools/recipe_modules/git/example.expected/cannot_fail_build.json +181 -0
  205. data/vendor/depot_tools/recipe_modules/git/example.expected/cat-file_test.json +199 -0
  206. data/vendor/depot_tools/recipe_modules/git/example.expected/count-objects_delta.json +250 -0
  207. data/vendor/depot_tools/recipe_modules/git/example.expected/count-objects_failed.json +181 -0
  208. data/vendor/depot_tools/recipe_modules/git/example.expected/count-objects_with_bad_output.json +182 -0
  209. data/vendor/depot_tools/recipe_modules/git/example.expected/count-objects_with_bad_output_fails_build.json +102 -0
  210. data/vendor/depot_tools/recipe_modules/git/example.expected/curl_trace_file.json +181 -0
  211. data/vendor/depot_tools/recipe_modules/git/example.expected/platform_win.json +186 -0
  212. data/vendor/depot_tools/recipe_modules/git/example.expected/rebase_failed.json +179 -0
  213. data/vendor/depot_tools/recipe_modules/git/example.expected/remote_not_origin.json +179 -0
  214. data/vendor/depot_tools/recipe_modules/git/example.expected/set_got_revision.json +178 -0
  215. data/vendor/depot_tools/recipe_modules/git/example.py +147 -0
  216. data/vendor/depot_tools/recipe_modules/git/resources/git_setup.py +61 -0
  217. data/vendor/depot_tools/recipe_modules/git/test_api.py +18 -0
  218. data/vendor/depot_tools/recipe_modules/git_cl/__init__.py +4 -0
  219. data/vendor/depot_tools/recipe_modules/git_cl/api.py +25 -0
  220. data/vendor/depot_tools/recipe_modules/git_cl/config.py +22 -0
  221. data/vendor/depot_tools/recipe_modules/git_cl/example.expected/basic.json +66 -0
  222. data/vendor/depot_tools/recipe_modules/git_cl/example.py +41 -0
  223. data/vendor/depot_tools/recipe_modules/infra_paths/__init__.py +4 -0
  224. data/vendor/depot_tools/recipe_modules/infra_paths/api.py +12 -0
  225. data/vendor/depot_tools/recipe_modules/infra_paths/example.expected/basic.json +14 -0
  226. data/vendor/depot_tools/recipe_modules/infra_paths/example.expected/paths_buildbot_linux.json +14 -0
  227. data/vendor/depot_tools/recipe_modules/infra_paths/example.expected/paths_buildbot_mac.json +14 -0
  228. data/vendor/depot_tools/recipe_modules/infra_paths/example.expected/paths_buildbot_win.json +14 -0
  229. data/vendor/depot_tools/recipe_modules/infra_paths/example.expected/paths_kitchen_linux.json +14 -0
  230. data/vendor/depot_tools/recipe_modules/infra_paths/example.expected/paths_kitchen_mac.json +14 -0
  231. data/vendor/depot_tools/recipe_modules/infra_paths/example.expected/paths_kitchen_win.json +14 -0
  232. data/vendor/depot_tools/recipe_modules/infra_paths/example.py +28 -0
  233. data/vendor/depot_tools/recipe_modules/infra_paths/path_config.py +45 -0
  234. data/vendor/depot_tools/recipe_modules/presubmit/__init__.py +4 -0
  235. data/vendor/depot_tools/recipe_modules/presubmit/api.py +20 -0
  236. data/vendor/depot_tools/recipe_modules/presubmit/example.expected/basic.json +18 -0
  237. data/vendor/depot_tools/recipe_modules/presubmit/example.py +15 -0
  238. data/vendor/depot_tools/recipe_modules/rietveld/__init__.py +5 -0
  239. data/vendor/depot_tools/recipe_modules/rietveld/api.py +94 -0
  240. data/vendor/depot_tools/recipe_modules/rietveld/example.expected/basic.json +30 -0
  241. data/vendor/depot_tools/recipe_modules/rietveld/example.py +24 -0
  242. data/vendor/depot_tools/recipe_modules/tryserver/__init__.py +15 -0
  243. data/vendor/depot_tools/recipe_modules/tryserver/api.py +280 -0
  244. data/vendor/depot_tools/recipe_modules/tryserver/example.expected/with_git_patch.json +104 -0
  245. data/vendor/depot_tools/recipe_modules/tryserver/example.expected/with_rietveld_patch.json +58 -0
  246. data/vendor/depot_tools/recipe_modules/tryserver/example.expected/with_rietveld_patch_new.json +58 -0
  247. data/vendor/depot_tools/recipe_modules/tryserver/example.expected/with_svn_patch.json +68 -0
  248. data/vendor/depot_tools/recipe_modules/tryserver/example.expected/with_wrong_patch.json +43 -0
  249. data/vendor/depot_tools/recipe_modules/tryserver/example.expected/with_wrong_patch_new.json +43 -0
  250. data/vendor/depot_tools/recipe_modules/tryserver/example.py +53 -0
  251. data/vendor/depot_tools/recipe_modules/tryserver/test_api.py +7 -0
  252. data/vendor/depot_tools/recipes.py +136 -0
  253. data/vendor/depot_tools/repo +1 -1
  254. data/vendor/depot_tools/rietveld.py +46 -15
  255. data/vendor/depot_tools/roll_dep.py +97 -36
  256. data/vendor/depot_tools/scm.py +3 -3
  257. data/vendor/depot_tools/setup_color.py +94 -0
  258. data/vendor/depot_tools/subprocess2.py +10 -1
  259. data/vendor/depot_tools/third_party/cq_client/OWNERS +0 -1
  260. data/vendor/depot_tools/third_party/cq_client/README.md +47 -9
  261. data/vendor/depot_tools/third_party/cq_client/cq.pb.go +617 -0
  262. data/vendor/depot_tools/third_party/cq_client/cq.proto +75 -17
  263. data/vendor/depot_tools/third_party/cq_client/cq_pb2.py +168 -41
  264. data/vendor/depot_tools/third_party/cq_client/testdata/cq_gerrit.cfg +55 -0
  265. data/vendor/depot_tools/third_party/cq_client/{test/cq_example.cfg → testdata/cq_rietveld.cfg} +14 -6
  266. data/vendor/depot_tools/third_party/fancy_urllib/README +5 -4
  267. data/vendor/depot_tools/third_party/fancy_urllib/__init__.py +114 -52
  268. data/vendor/depot_tools/third_party/protobuf26/README.chromium +9 -6
  269. data/vendor/depot_tools/third_party/upload.py +17 -31
  270. data/vendor/depot_tools/trychange.py +0 -2
  271. data/vendor/depot_tools/update_depot_tools +29 -11
  272. data/vendor/depot_tools/update_depot_tools.bat +4 -9
  273. data/vendor/depot_tools/upload_to_google_storage.py +42 -5
  274. data/vendor/depot_tools/win_toolchain/OWNERS +1 -0
  275. data/vendor/depot_tools/win_toolchain/get_toolchain_if_necessary.py +227 -52
  276. data/vendor/depot_tools/win_toolchain/package_from_installed.py +203 -88
  277. metadata +161 -81
  278. data/patches/arm/do-not-imply-vfp3-and-armv7.patch +0 -16
  279. data/patches/arm/do-not-use-vfp2.patch +0 -13
  280. data/patches/clang51/no-unused-variable.patch +0 -12
  281. data/vendor/depot_tools/bootstrap/.gitignore +0 -2
  282. data/vendor/depot_tools/bootstrap/bootstrap.py +0 -234
  283. data/vendor/depot_tools/bootstrap/deps.pyl +0 -15
  284. data/vendor/depot_tools/bootstrap/util.py +0 -87
  285. data/vendor/depot_tools/bootstrap/virtualenv/.gitignore +0 -10
  286. data/vendor/depot_tools/bootstrap/virtualenv/.travis.yml +0 -28
  287. data/vendor/depot_tools/bootstrap/virtualenv/AUTHORS.txt +0 -91
  288. data/vendor/depot_tools/bootstrap/virtualenv/CONTRIBUTING.rst +0 -21
  289. data/vendor/depot_tools/bootstrap/virtualenv/LICENSE.txt +0 -22
  290. data/vendor/depot_tools/bootstrap/virtualenv/MANIFEST.in +0 -11
  291. data/vendor/depot_tools/bootstrap/virtualenv/README.rst +0 -10
  292. data/vendor/depot_tools/bootstrap/virtualenv/bin/rebuild-script.py +0 -71
  293. data/vendor/depot_tools/bootstrap/virtualenv/docs/changes.rst +0 -747
  294. data/vendor/depot_tools/bootstrap/virtualenv/docs/conf.py +0 -149
  295. data/vendor/depot_tools/bootstrap/virtualenv/docs/development.rst +0 -61
  296. data/vendor/depot_tools/bootstrap/virtualenv/docs/index.rst +0 -137
  297. data/vendor/depot_tools/bootstrap/virtualenv/docs/installation.rst +0 -58
  298. data/vendor/depot_tools/bootstrap/virtualenv/docs/make.bat +0 -170
  299. data/vendor/depot_tools/bootstrap/virtualenv/docs/reference.rst +0 -256
  300. data/vendor/depot_tools/bootstrap/virtualenv/docs/userguide.rst +0 -249
  301. data/vendor/depot_tools/bootstrap/virtualenv/scripts/virtualenv +0 -3
  302. data/vendor/depot_tools/bootstrap/virtualenv/setup.py +0 -111
  303. data/vendor/depot_tools/bootstrap/virtualenv/tests/test_activate.sh +0 -94
  304. data/vendor/depot_tools/bootstrap/virtualenv/tests/test_activate_expected.output +0 -2
  305. data/vendor/depot_tools/bootstrap/virtualenv/tests/test_virtualenv.py +0 -139
  306. data/vendor/depot_tools/bootstrap/virtualenv/tests/tox.ini +0 -12
  307. data/vendor/depot_tools/bootstrap/virtualenv/tox.ini +0 -17
  308. data/vendor/depot_tools/bootstrap/virtualenv/virtualenv.py +0 -2367
  309. data/vendor/depot_tools/bootstrap/virtualenv/virtualenv_embedded/activate.bat +0 -26
  310. data/vendor/depot_tools/bootstrap/virtualenv/virtualenv_embedded/activate.csh +0 -42
  311. data/vendor/depot_tools/bootstrap/virtualenv/virtualenv_embedded/activate.fish +0 -74
  312. data/vendor/depot_tools/bootstrap/virtualenv/virtualenv_embedded/activate.ps1 +0 -150
  313. data/vendor/depot_tools/bootstrap/virtualenv/virtualenv_embedded/activate.sh +0 -80
  314. data/vendor/depot_tools/bootstrap/virtualenv/virtualenv_embedded/activate_this.py +0 -34
  315. data/vendor/depot_tools/bootstrap/virtualenv/virtualenv_embedded/deactivate.bat +0 -20
  316. data/vendor/depot_tools/bootstrap/virtualenv/virtualenv_embedded/distutils-init.py +0 -101
  317. data/vendor/depot_tools/bootstrap/virtualenv/virtualenv_embedded/distutils.cfg +0 -6
  318. data/vendor/depot_tools/bootstrap/virtualenv/virtualenv_embedded/site.py +0 -758
  319. data/vendor/depot_tools/bootstrap/virtualenv/virtualenv_support/pip-6.0-py2.py3-none-any.whl +0 -0
  320. data/vendor/depot_tools/bootstrap/virtualenv/virtualenv_support/setuptools-8.2.1-py2.py3-none-any.whl +0 -0
  321. data/vendor/depot_tools/bootstrap/win/README.google +0 -16
  322. data/vendor/depot_tools/cbuildbot +0 -96
  323. data/vendor/depot_tools/chrome_set_ver +0 -96
  324. data/vendor/depot_tools/cros +0 -96
  325. data/vendor/depot_tools/cros_sdk +0 -96
  326. data/vendor/depot_tools/git-cl-upload-hook +0 -52
  327. data/vendor/depot_tools/git-crup +0 -45
  328. data/vendor/depot_tools/python_git_runner.sh +0 -36
  329. data/vendor/depot_tools/recipes/blink.py +0 -59
  330. data/vendor/depot_tools/third_party/cq_client/test/validate_config_test.py +0 -52
  331. data/vendor/depot_tools/third_party/cq_client/validate_config.py +0 -108
  332. data/vendor/depot_tools/win_toolchain/toolchain2013.py +0 -494
@@ -0,0 +1,51 @@
1
+ [
2
+ {
3
+ "cmd": [
4
+ "python",
5
+ "-u",
6
+ "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
7
+ "--master",
8
+ "chromium.testing.master",
9
+ "--builder",
10
+ "TestBuilder",
11
+ "--slave",
12
+ "TestSlavename",
13
+ "--spec",
14
+ "cache_dir = '[GIT_CACHE]'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': True, 'name': 'src', 'url': 'svn://svn.chromium.org/chrome/trunk/src'}]",
15
+ "--root",
16
+ "src/v8",
17
+ "--revision_mapping_file",
18
+ "{\"src\": \"got_cr_revision\"}",
19
+ "--git-cache-dir",
20
+ "[GIT_CACHE]",
21
+ "--issue",
22
+ "12853011",
23
+ "--patchset",
24
+ "1",
25
+ "--rietveld_server",
26
+ "https://codereview.chromium.org",
27
+ "--output_json",
28
+ "/path/to/tmp/json",
29
+ "--revision",
30
+ "src@HEAD",
31
+ "--revision",
32
+ "src/v8@HEAD"
33
+ ],
34
+ "env": {
35
+ "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]"
36
+ },
37
+ "name": "bot_update",
38
+ "~followup_annotations": [
39
+ "@@@STEP_LOG_LINE@json.output@{@@@",
40
+ "@@@STEP_LOG_LINE@json.output@ \"did_run\": false, @@@",
41
+ "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false@@@",
42
+ "@@@STEP_LOG_LINE@json.output@}@@@",
43
+ "@@@STEP_LOG_END@json.output@@@"
44
+ ]
45
+ },
46
+ {
47
+ "name": "$result",
48
+ "recipe_result": null,
49
+ "status_code": 0
50
+ }
51
+ ]
@@ -0,0 +1,172 @@
1
+ # Copyright 2014 The Chromium Authors. All rights reserved.
2
+ # Use of this source code is governed by a BSD-style license that can be
3
+ # found in the LICENSE file.
4
+
5
+ DEPS = [
6
+ 'bot_update',
7
+ 'gclient',
8
+ 'recipe_engine/path',
9
+ 'recipe_engine/properties',
10
+ ]
11
+
12
+ def RunSteps(api):
13
+ api.gclient.use_mirror = True
14
+
15
+ src_cfg = api.gclient.make_config(GIT_MODE=True, CACHE_DIR='[GIT_CACHE]')
16
+ soln = src_cfg.solutions.add()
17
+ soln.name = 'src'
18
+ soln.url = 'svn://svn.chromium.org/chrome/trunk/src'
19
+ soln.revision = api.properties.get('revision')
20
+ api.gclient.c = src_cfg
21
+ api.gclient.c.revisions.update(api.properties.get('revisions', {}))
22
+ api.gclient.c.got_revision_mapping['src'] = 'got_cr_revision'
23
+ api.gclient.c.patch_projects['v8'] = ('src/v8', 'HEAD')
24
+ api.gclient.c.patch_projects['angle/angle'] = ('src/third_party/angle',
25
+ 'HEAD')
26
+ patch = api.properties.get('patch', True)
27
+ clobber = True if api.properties.get('clobber') else False
28
+ force = True if api.properties.get('force') else False
29
+ no_shallow = True if api.properties.get('no_shallow') else False
30
+ output_manifest = api.properties.get('output_manifest', False)
31
+ with_branch_heads = api.properties.get('with_branch_heads', False)
32
+ refs = api.properties.get('refs', [])
33
+ oauth2 = api.properties.get('oauth2', False)
34
+ root_solution_revision = api.properties.get('root_solution_revision')
35
+ suffix = api.properties.get('suffix')
36
+ gerrit_no_reset = True if api.properties.get('gerrit_no_reset') else False
37
+ api.bot_update.ensure_checkout(force=force,
38
+ no_shallow=no_shallow,
39
+ patch=patch,
40
+ with_branch_heads=with_branch_heads,
41
+ output_manifest=output_manifest,
42
+ refs=refs, patch_oauth2=oauth2,
43
+ clobber=clobber,
44
+ root_solution_revision=root_solution_revision,
45
+ suffix=suffix,
46
+ gerrit_no_reset=gerrit_no_reset)
47
+
48
+
49
+ def GenTests(api):
50
+ yield api.test('basic') + api.properties(
51
+ mastername='chromium.linux',
52
+ buildername='Linux Builder',
53
+ slavename='totallyaslave-m1',
54
+ patch=False,
55
+ revision='abc'
56
+ )
57
+ yield api.test('basic_with_branch_heads') + api.properties(
58
+ mastername='chromium.linux',
59
+ buildername='Linux Builder',
60
+ slavename='totallyaslave-m1',
61
+ with_branch_heads=True,
62
+ suffix='with branch heads'
63
+ )
64
+ yield api.test('basic_output_manifest') + api.properties(
65
+ mastername='chromium.linux',
66
+ buildername='Linux Builder',
67
+ slavename='totallyaslave-m1',
68
+ output_manifest=True,
69
+ )
70
+ yield api.test('tryjob') + api.properties(
71
+ mastername='tryserver.chromium.linux',
72
+ buildername='linux_rel',
73
+ slavename='totallyaslave-c4',
74
+ issue=12345,
75
+ patchset=654321,
76
+ patch_url='http://src.chromium.org/foo/bar'
77
+ )
78
+ yield api.test('trychange') + api.properties(
79
+ mastername='tryserver.chromium.linux',
80
+ buildername='linux_rel',
81
+ slavename='totallyaslave-c4',
82
+ refs=['+refs/change/1/2/333'],
83
+ )
84
+ yield api.test('trychange_oauth2') + api.properties(
85
+ mastername='tryserver.chromium.linux',
86
+ buildername='linux_rel',
87
+ slavename='totallyaslave-c4',
88
+ oauth2=True,
89
+ )
90
+ yield api.test('tryjob_fail') + api.properties(
91
+ mastername='tryserver.chromium.linux',
92
+ buildername='linux_rel',
93
+ slavename='totallyaslave-c4',
94
+ issue=12345,
95
+ patchset=654321,
96
+ patch_url='http://src.chromium.org/foo/bar',
97
+ ) + api.step_data('bot_update', retcode=1)
98
+ yield api.test('tryjob_fail_patch') + api.properties(
99
+ mastername='tryserver.chromium.linux',
100
+ buildername='linux_rel',
101
+ slavename='totallyaslave-c4',
102
+ issue=12345,
103
+ patchset=654321,
104
+ patch_url='http://src.chromium.org/foo/bar',
105
+ fail_patch='apply',
106
+ ) + api.step_data('bot_update', retcode=88)
107
+ yield api.test('tryjob_fail_patch_download') + api.properties(
108
+ mastername='tryserver.chromium.linux',
109
+ buildername='linux_rel',
110
+ slavename='totallyaslave-c4',
111
+ issue=12345,
112
+ patchset=654321,
113
+ patch_url='http://src.chromium.org/foo/bar',
114
+ fail_patch='download'
115
+ ) + api.step_data('bot_update', retcode=87)
116
+ yield api.test('forced') + api.properties(
117
+ mastername='experimental',
118
+ buildername='Experimental Builder',
119
+ slavename='somehost',
120
+ force=1
121
+ )
122
+ yield api.test('no_shallow') + api.properties(
123
+ mastername='experimental',
124
+ buildername='Experimental Builder',
125
+ slavename='somehost',
126
+ no_shallow=1
127
+ )
128
+ yield api.test('off') + api.properties(
129
+ mastername='experimental',
130
+ buildername='Experimental Builder',
131
+ slavename='somehost',
132
+ )
133
+ yield api.test('svn_mode') + api.properties(
134
+ mastername='experimental.svn',
135
+ buildername='Experimental SVN Builder',
136
+ slavename='somehost',
137
+ force=1
138
+ )
139
+ yield api.test('clobber') + api.properties(
140
+ mastername='experimental',
141
+ buildername='Experimental Builder',
142
+ slavename='somehost',
143
+ clobber=1
144
+ )
145
+ yield api.test('reset_root_solution_revision') + api.properties(
146
+ mastername='experimental',
147
+ buildername='Experimental Builder',
148
+ slavename='somehost',
149
+ root_solution_revision='revision',
150
+ )
151
+ yield api.test('gerrit_no_reset') + api.properties(
152
+ mastername='experimental',
153
+ buildername='Experimental Builder',
154
+ slavename='somehost',
155
+ gerrit_no_reset=1
156
+ )
157
+ yield api.test('tryjob_v8') + api.properties(
158
+ mastername='tryserver.chromium.linux',
159
+ buildername='linux_rel',
160
+ slavename='totallyaslave-c4',
161
+ issue=12345,
162
+ patchset=654321,
163
+ patch_url='http://src.chromium.org/foo/bar',
164
+ patch_project='v8',
165
+ revisions={'src/v8': 'abc'}
166
+ )
167
+ yield api.test('tryjob_v8_head_by_default') + api.properties.tryserver(
168
+ patch_project='v8',
169
+ )
170
+ yield api.test('tryjob_gerrit_angle') + api.properties.tryserver_gerrit(
171
+ full_project_name='angle/angle',
172
+ )
@@ -0,0 +1,1764 @@
1
+ #!/usr/bin/env python
2
+ # Copyright 2014 The Chromium Authors. All rights reserved.
3
+ # Use of this source code is governed by a BSD-style license that can be
4
+ # found in the LICENSE file.
5
+
6
+ # TODO(hinoka): Use logging.
7
+
8
+ import cStringIO
9
+ import codecs
10
+ import collections
11
+ import copy
12
+ import ctypes
13
+ import json
14
+ import optparse
15
+ import os
16
+ import pprint
17
+ import random
18
+ import re
19
+ import socket
20
+ import subprocess
21
+ import sys
22
+ import tempfile
23
+ import threading
24
+ import time
25
+ import urllib2
26
+ import urlparse
27
+ import uuid
28
+
29
+ import os.path as path
30
+
31
+ # How many bytes at a time to read from pipes.
32
+ BUF_SIZE = 256
33
+
34
+
35
+ # TODO(luqui): This is a horrible hack to identify build_internal when build
36
+ # is a recipe dependency. bot_update should not be depending on internal,
37
+ # rather the arrow should go the other way (or just be destroyed).
38
+ def check_dir(name, dirs, default=None):
39
+ for d in dirs:
40
+ d = path.abspath(d)
41
+ if path.basename(d) == name and path.isdir(d):
42
+ return d
43
+ return default
44
+
45
+
46
+ # Define a bunch of directory paths.
47
+ # Relative to the current working directory.
48
+ CURRENT_DIR = path.abspath(os.getcwd())
49
+ BUILDER_DIR = path.dirname(CURRENT_DIR)
50
+ SLAVE_DIR = path.dirname(BUILDER_DIR)
51
+
52
+ # Relative to this script's filesystem path.
53
+ THIS_DIR = path.dirname(path.abspath(__file__))
54
+ SCRIPTS_DIR = check_dir(
55
+ 'scripts', [
56
+ path.dirname(THIS_DIR),
57
+ path.join(SLAVE_DIR, '..', 'scripts'),
58
+ path.join(THIS_DIR, # resources
59
+ '..', # bot_update
60
+ '..', # recipe_modules
61
+ '..', # depot_tools
62
+ '..', # .recipe_deps
63
+ '..', # slave
64
+ '..', # scripts
65
+ '..', # build_internal
66
+ '..', # ROOT_DIR
67
+ 'build',
68
+ 'scripts'),
69
+ path.join(SLAVE_DIR, '..', 'build', 'scripts'),
70
+ ], default=path.dirname(THIS_DIR))
71
+ BUILD_DIR = path.dirname(SCRIPTS_DIR)
72
+ ROOT_DIR = path.dirname(BUILD_DIR)
73
+
74
+ DEPOT_TOOLS_DIR = path.abspath(path.join(THIS_DIR, '..', '..', '..'))
75
+
76
+ BUILD_INTERNAL_DIR = check_dir(
77
+ 'build_internal', [
78
+ path.join(ROOT_DIR, 'build_internal'),
79
+ path.join(ROOT_DIR, # .recipe_deps
80
+ path.pardir, # slave
81
+ path.pardir, # scripts
82
+ path.pardir), # build_internal
83
+ ])
84
+
85
+
86
+ CHROMIUM_GIT_HOST = 'https://chromium.googlesource.com'
87
+ CHROMIUM_SRC_URL = CHROMIUM_GIT_HOST + '/chromium/src.git'
88
+
89
+ # Official builds use buildspecs, so this is a special case.
90
+ BUILDSPEC_TYPE = collections.namedtuple('buildspec',
91
+ ('container', 'version'))
92
+ BUILDSPEC_RE = (r'^/chrome-internal/trunk/tools/buildspec/'
93
+ '(build|branches|releases)/(.+)$')
94
+ GIT_BUILDSPEC_PATH = ('https://chrome-internal.googlesource.com/chrome/tools/'
95
+ 'buildspec')
96
+ BRANCH_HEADS_REFSPEC = '+refs/branch-heads/*'
97
+
98
+ BUILDSPEC_COMMIT_RE = (
99
+ re.compile(r'Buildspec for.*version (\d+\.\d+\.\d+\.\d+)'),
100
+ re.compile(r'Create (\d+\.\d+\.\d+\.\d+) buildspec'),
101
+ re.compile(r'Auto-converted (\d+\.\d+\.\d+\.\d+) buildspec to git'),
102
+ )
103
+
104
+ # Regular expression that matches a single commit footer line.
105
+ COMMIT_FOOTER_ENTRY_RE = re.compile(r'([^:]+):\s+(.+)')
106
+
107
+ # Footer metadata keys for regular and gsubtreed mirrored commit positions.
108
+ COMMIT_POSITION_FOOTER_KEY = 'Cr-Commit-Position'
109
+ COMMIT_ORIGINAL_POSITION_FOOTER_KEY = 'Cr-Original-Commit-Position'
110
+ # Regular expression to parse a commit position
111
+ COMMIT_POSITION_RE = re.compile(r'(.+)@\{#(\d+)\}')
112
+
113
+ # Regular expression to parse gclient's revinfo entries.
114
+ REVINFO_RE = re.compile(r'^([^:]+):\s+([^@]+)@(.+)$')
115
+
116
+ # Used by 'ResolveSvnRevisionFromGitiles'
117
+ GIT_SVN_PROJECT_MAP = {
118
+ 'webkit': {
119
+ 'svn_url': 'svn://svn.chromium.org/blink',
120
+ 'branch_map': [
121
+ (r'trunk', r'refs/heads/master'),
122
+ (r'branches/([^/]+)', r'refs/branch-heads/\1'),
123
+ ],
124
+ },
125
+ 'v8': {
126
+ 'svn_url': 'https://v8.googlecode.com/svn',
127
+ 'branch_map': [
128
+ (r'trunk', r'refs/heads/candidates'),
129
+ (r'branches/bleeding_edge', r'refs/heads/master'),
130
+ (r'branches/([^/]+)', r'refs/branch-heads/\1'),
131
+ ],
132
+ },
133
+ 'nacl': {
134
+ 'svn_url': 'svn://svn.chromium.org/native_client',
135
+ 'branch_map': [
136
+ (r'trunk/src/native_client', r'refs/heads/master'),
137
+ ],
138
+ },
139
+ }
140
+
141
+ # Key for the 'git-svn' ID metadata commit footer entry.
142
+ GIT_SVN_ID_FOOTER_KEY = 'git-svn-id'
143
+ # e.g., git-svn-id: https://v8.googlecode.com/svn/trunk@23117
144
+ # ce2b1a6d-e550-0410-aec6-3dcde31c8c00
145
+ GIT_SVN_ID_RE = re.compile(r'((?:\w+)://[^@]+)@(\d+)\s+(?:[a-zA-Z0-9\-]+)')
146
+
147
+
148
+ # This is the git mirror of the buildspecs repository. We could rely on the svn
149
+ # checkout, now that the git buildspecs are checked in alongside the svn
150
+ # buildspecs, but we're going to want to pull all the buildspecs from here
151
+ # eventually anyhow, and there's already some logic to pull from git (for the
152
+ # old git_buildspecs.git repo), so just stick with that.
153
+ GIT_BUILDSPEC_REPO = (
154
+ 'https://chrome-internal.googlesource.com/chrome/tools/buildspec')
155
+
156
+ # Copied from scripts/recipes/chromium.py.
157
+ GOT_REVISION_MAPPINGS = {
158
+ '/chrome/trunk/src': {
159
+ 'src/': 'got_revision',
160
+ 'src/native_client/': 'got_nacl_revision',
161
+ 'src/tools/swarm_client/': 'got_swarm_client_revision',
162
+ 'src/tools/swarming_client/': 'got_swarming_client_revision',
163
+ 'src/third_party/WebKit/': 'got_webkit_revision',
164
+ 'src/third_party/webrtc/': 'got_webrtc_revision',
165
+ 'src/v8/': 'got_v8_revision',
166
+ }
167
+ }
168
+
169
+
170
+ BOT_UPDATE_MESSAGE = """
171
+ What is the "Bot Update" step?
172
+ ==============================
173
+
174
+ This step ensures that the source checkout on the bot (e.g. Chromium's src/ and
175
+ its dependencies) is checked out in a consistent state. This means that all of
176
+ the necessary repositories are checked out, no extra repositories are checked
177
+ out, and no locally modified files are present.
178
+
179
+ These actions used to be taken care of by the "gclient revert" and "update"
180
+ steps. However, those steps are known to be buggy and occasionally flaky. This
181
+ step has two main advantages over them:
182
+ * it only operates in Git, so the logic can be clearer and cleaner; and
183
+ * it is a slave-side script, so its behavior can be modified without
184
+ restarting the master.
185
+
186
+ Why Git, you ask? Because that is the direction that the Chromium project is
187
+ heading. This step is an integral part of the transition from using the SVN repo
188
+ at chrome/trunk/src to using the Git repo src.git. Please pardon the dust while
189
+ we fully convert everything to Git. This message will get out of your way
190
+ eventually, and the waterfall will be a happier place because of it.
191
+
192
+ This step can be activated or deactivated independently on every builder on
193
+ every master. When it is active, the "gclient revert" and "update" steps become
194
+ no-ops. When it is inactive, it prints this message, cleans up after itself, and
195
+ lets everything else continue as though nothing has changed. Eventually, when
196
+ everything is stable enough, this step will replace them entirely.
197
+
198
+ Debugging information:
199
+ (master/builder/slave may be unspecified on recipes)
200
+ master: %(master)s
201
+ builder: %(builder)s
202
+ slave: %(slave)s
203
+ forced by recipes: %(recipe)s
204
+ CURRENT_DIR: %(CURRENT_DIR)s
205
+ BUILDER_DIR: %(BUILDER_DIR)s
206
+ SLAVE_DIR: %(SLAVE_DIR)s
207
+ THIS_DIR: %(THIS_DIR)s
208
+ SCRIPTS_DIR: %(SCRIPTS_DIR)s
209
+ BUILD_DIR: %(BUILD_DIR)s
210
+ ROOT_DIR: %(ROOT_DIR)s
211
+ DEPOT_TOOLS_DIR: %(DEPOT_TOOLS_DIR)s
212
+ bot_update.py is:"""
213
+
214
+ ACTIVATED_MESSAGE = """ACTIVE.
215
+ The bot will perform a Git checkout in this step.
216
+ The "gclient revert" and "update" steps are no-ops.
217
+
218
+ """
219
+
220
+ NOT_ACTIVATED_MESSAGE = """INACTIVE.
221
+ This step does nothing. You actually want to look at the "update" step.
222
+
223
+ """
224
+
225
+
226
+ GCLIENT_TEMPLATE = """solutions = %(solutions)s
227
+
228
+ cache_dir = r%(cache_dir)s
229
+ %(target_os)s
230
+ %(target_os_only)s
231
+ """
232
+
233
+
234
+ internal_data = {}
235
+ if BUILD_INTERNAL_DIR:
236
+ local_vars = {}
237
+ try:
238
+ execfile(os.path.join(
239
+ BUILD_INTERNAL_DIR, 'scripts', 'slave', 'bot_update_cfg.py'),
240
+ local_vars)
241
+ except Exception:
242
+ # Same as if BUILD_INTERNAL_DIR didn't exist in the first place.
243
+ print 'Warning: unable to read internal configuration file.'
244
+ print 'If this is an internal bot, this step may be erroneously inactive.'
245
+ internal_data = local_vars
246
+
247
+ RECOGNIZED_PATHS = {
248
+ # If SVN path matches key, the entire URL is rewritten to the Git url.
249
+ '/chrome/trunk/src':
250
+ CHROMIUM_SRC_URL,
251
+ '/chrome/trunk/src/tools/cros.DEPS':
252
+ CHROMIUM_GIT_HOST + '/chromium/src/tools/cros.DEPS.git',
253
+ }
254
+ RECOGNIZED_PATHS.update(internal_data.get('RECOGNIZED_PATHS', {}))
255
+
256
+ ENABLED_MASTERS = [
257
+ 'bot_update.always_on',
258
+ 'chromium.android',
259
+ 'chromium.angle',
260
+ 'chromium.chrome',
261
+ 'chromium.chromedriver',
262
+ 'chromium.chromiumos',
263
+ 'chromium',
264
+ 'chromium.fyi',
265
+ 'chromium.goma',
266
+ 'chromium.gpu',
267
+ 'chromium.gpu.fyi',
268
+ 'chromium.infra',
269
+ 'chromium.infra.cron',
270
+ 'chromium.linux',
271
+ 'chromium.lkgr',
272
+ 'chromium.mac',
273
+ 'chromium.memory',
274
+ 'chromium.memory.fyi',
275
+ 'chromium.perf',
276
+ 'chromium.perf.fyi',
277
+ 'chromium.swarm',
278
+ 'chromium.webkit',
279
+ 'chromium.webrtc',
280
+ 'chromium.webrtc.fyi',
281
+ 'chromium.win',
282
+ 'client.catapult',
283
+ 'client.drmemory',
284
+ 'client.mojo',
285
+ 'client.nacl',
286
+ 'client.nacl.ports',
287
+ 'client.nacl.sdk',
288
+ 'client.nacl.toolchain',
289
+ 'client.pdfium',
290
+ 'client.skia',
291
+ 'client.skia.fyi',
292
+ 'client.v8',
293
+ 'client.v8.branches',
294
+ 'client.v8.fyi',
295
+ 'client.v8.ports',
296
+ 'client.webrtc',
297
+ 'client.webrtc.fyi',
298
+ 'tryserver.blink',
299
+ 'tryserver.client.catapult',
300
+ 'tryserver.client.mojo',
301
+ 'tryserver.chromium.android',
302
+ 'tryserver.chromium.angle',
303
+ 'tryserver.chromium.linux',
304
+ 'tryserver.chromium.mac',
305
+ 'tryserver.chromium.perf',
306
+ 'tryserver.chromium.win',
307
+ 'tryserver.infra',
308
+ 'tryserver.nacl',
309
+ 'tryserver.v8',
310
+ 'tryserver.webrtc',
311
+ ]
312
+ ENABLED_MASTERS += internal_data.get('ENABLED_MASTERS', [])
313
+
314
+ ENABLED_BUILDERS = {
315
+ 'client.dart.fyi': [
316
+ 'v8-linux-release',
317
+ 'v8-mac-release',
318
+ 'v8-win-release',
319
+ ],
320
+ 'client.dynamorio': [
321
+ 'linux-v8-dr',
322
+ ],
323
+ }
324
+ ENABLED_BUILDERS.update(internal_data.get('ENABLED_BUILDERS', {}))
325
+
326
+ ENABLED_SLAVES = {}
327
+ ENABLED_SLAVES.update(internal_data.get('ENABLED_SLAVES', {}))
328
+
329
+ # Disabled filters get run AFTER enabled filters, so for example if a builder
330
+ # config is enabled, but a bot on that builder is disabled, that bot will
331
+ # be disabled.
332
+ DISABLED_BUILDERS = {}
333
+ DISABLED_BUILDERS.update(internal_data.get('DISABLED_BUILDERS', {}))
334
+
335
+ DISABLED_SLAVES = {}
336
+ DISABLED_SLAVES.update(internal_data.get('DISABLED_SLAVES', {}))
337
+
338
+ # These masters work only in Git, meaning for got_revision, always output
339
+ # a git hash rather than a SVN rev.
340
+ GIT_MASTERS = [
341
+ 'client.v8',
342
+ 'client.v8.branches',
343
+ 'client.v8.ports',
344
+ 'tryserver.v8',
345
+ ]
346
+ GIT_MASTERS += internal_data.get('GIT_MASTERS', [])
347
+
348
+
349
+ # How many times to try before giving up.
350
+ ATTEMPTS = 5
351
+
352
+ # Find deps2git
353
+ DEPS2GIT_DIR_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git')
354
+ DEPS2GIT_PATH = path.join(DEPS2GIT_DIR_PATH, 'deps2git.py')
355
+ S2G_INTERNAL_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git_internal',
356
+ 'svn_to_git_internal.py')
357
+ GIT_CACHE_PATH = path.join(DEPOT_TOOLS_DIR, 'git_cache.py')
358
+
359
+ # Find the patch tool.
360
+ if sys.platform.startswith('win'):
361
+ if not BUILD_INTERNAL_DIR:
362
+ print 'Warning: could not find patch tool because there is no '
363
+ print 'build_internal present.'
364
+ PATCH_TOOL = None
365
+ else:
366
+ PATCH_TOOL = path.join(BUILD_INTERNAL_DIR, 'tools', 'patch.EXE')
367
+ else:
368
+ PATCH_TOOL = '/usr/bin/patch'
369
+
370
+ # If there is less than 100GB of disk space on the system, then we do
371
+ # a shallow checkout.
372
+ SHALLOW_CLONE_THRESHOLD = 100 * 1024 * 1024 * 1024
373
+
374
+
375
+ class SubprocessFailed(Exception):
376
+ def __init__(self, message, code, output):
377
+ Exception.__init__(self, message)
378
+ self.code = code
379
+ self.output = output
380
+
381
+
382
+ class PatchFailed(SubprocessFailed):
383
+ pass
384
+
385
+
386
+ class GclientSyncFailed(SubprocessFailed):
387
+ pass
388
+
389
+
390
+ class SVNRevisionNotFound(Exception):
391
+ pass
392
+
393
+
394
+ class InvalidDiff(Exception):
395
+ pass
396
+
397
+
398
+ class Inactive(Exception):
399
+ """Not really an exception, just used to exit early cleanly."""
400
+ pass
401
+
402
+
403
+ RETRY = object()
404
+ OK = object()
405
+ FAIL = object()
406
+
407
+
408
+ class PsPrinter(object):
409
+ def __init__(self, interval=300):
410
+ self.interval = interval
411
+ self.active = sys.platform.startswith('linux2')
412
+ self.thread = None
413
+
414
+ @staticmethod
415
+ def print_pstree():
416
+ """Debugging function used to print "ps auxwwf" for stuck processes."""
417
+ subprocess.call(['ps', 'auxwwf'])
418
+
419
+ def poke(self):
420
+ if self.active:
421
+ self.cancel()
422
+ self.thread = threading.Timer(self.interval, self.print_pstree)
423
+ self.thread.start()
424
+
425
+ def cancel(self):
426
+ if self.active and self.thread is not None:
427
+ self.thread.cancel()
428
+ self.thread = None
429
+
430
+
431
+ def call(*args, **kwargs): # pragma: no cover
432
+ """Interactive subprocess call."""
433
+ kwargs['stdout'] = subprocess.PIPE
434
+ kwargs['stderr'] = subprocess.STDOUT
435
+ kwargs.setdefault('bufsize', BUF_SIZE)
436
+ cwd = kwargs.get('cwd', os.getcwd())
437
+ result_fn = kwargs.pop('result_fn', lambda code, out: RETRY if code else OK)
438
+ stdin_data = kwargs.pop('stdin_data', None)
439
+ tries = kwargs.pop('tries', ATTEMPTS)
440
+ if stdin_data:
441
+ kwargs['stdin'] = subprocess.PIPE
442
+ out = cStringIO.StringIO()
443
+ new_env = kwargs.get('env', {})
444
+ env = copy.copy(os.environ)
445
+ env.update(new_env)
446
+ kwargs['env'] = env
447
+ attempt = 0
448
+ for attempt in range(1, tries + 1):
449
+ attempt_msg = ' (attempt #%d)' % attempt if attempt else ''
450
+ if new_env:
451
+ print '===Injecting Environment Variables==='
452
+ for k, v in sorted(new_env.items()):
453
+ print '%s: %s' % (k, v)
454
+ print '===Running %s%s===' % (' '.join(args), attempt_msg)
455
+ print 'In directory: %s' % cwd
456
+ start_time = time.time()
457
+ proc = subprocess.Popen(args, **kwargs)
458
+ if stdin_data:
459
+ proc.stdin.write(stdin_data)
460
+ proc.stdin.close()
461
+ psprinter = PsPrinter()
462
+ # This is here because passing 'sys.stdout' into stdout for proc will
463
+ # produce out of order output.
464
+ hanging_cr = False
465
+ while True:
466
+ psprinter.poke()
467
+ buf = proc.stdout.read(BUF_SIZE)
468
+ if not buf:
469
+ break
470
+ if hanging_cr:
471
+ buf = '\r' + buf
472
+ hanging_cr = buf.endswith('\r')
473
+ if hanging_cr:
474
+ buf = buf[:-1]
475
+ buf = buf.replace('\r\n', '\n').replace('\r', '\n')
476
+ sys.stdout.write(buf)
477
+ out.write(buf)
478
+ if hanging_cr:
479
+ sys.stdout.write('\n')
480
+ out.write('\n')
481
+ psprinter.cancel()
482
+
483
+ code = proc.wait()
484
+ elapsed_time = ((time.time() - start_time) / 60.0)
485
+ outval = out.getvalue()
486
+ result = result_fn(code, outval)
487
+ if result in (FAIL, RETRY):
488
+ print '===Failed in %.1f mins===' % elapsed_time
489
+ print
490
+ else:
491
+ print '===Succeeded in %.1f mins===' % elapsed_time
492
+ print
493
+ return outval
494
+ if result is FAIL:
495
+ break
496
+ if result is RETRY and attempt < tries:
497
+ sleep_backoff = 4 ** attempt
498
+ sleep_time = random.randint(sleep_backoff, int(sleep_backoff * 1.2))
499
+ print '===backing off, sleeping for %d secs===' % sleep_time
500
+ time.sleep(sleep_time)
501
+
502
+ raise SubprocessFailed('%s failed with code %d in %s after %d attempts.' %
503
+ (' '.join(args), code, cwd, attempt),
504
+ code, outval)
505
+
506
+
507
+ def git(*args, **kwargs): # pragma: no cover
508
+ """Wrapper around call specifically for Git commands."""
509
+ if args and args[0] == 'cache':
510
+ # Rewrite "git cache" calls into "python git_cache.py".
511
+ cmd = (sys.executable, '-u', GIT_CACHE_PATH) + args[1:]
512
+ else:
513
+ git_executable = 'git'
514
+ # On windows, subprocess doesn't fuzzy-match 'git' to 'git.bat', so we
515
+ # have to do it explicitly. This is better than passing shell=True.
516
+ if sys.platform.startswith('win'):
517
+ git_executable += '.bat'
518
+ cmd = (git_executable,) + args
519
+ return call(*cmd, **kwargs)
520
+
521
+
522
+ def get_gclient_spec(solutions, target_os, target_os_only, git_cache_dir):
523
+ return GCLIENT_TEMPLATE % {
524
+ 'solutions': pprint.pformat(solutions, indent=4),
525
+ 'cache_dir': '"%s"' % git_cache_dir,
526
+ 'target_os': ('\ntarget_os=%s' % target_os) if target_os else '',
527
+ 'target_os_only': '\ntarget_os_only=%s' % target_os_only
528
+ }
529
+
530
+
531
+ def check_enabled(master, builder, slave):
532
+ if master in ENABLED_MASTERS:
533
+ return True
534
+ builder_list = ENABLED_BUILDERS.get(master)
535
+ if builder_list and builder in builder_list:
536
+ return True
537
+ slave_list = ENABLED_SLAVES.get(master)
538
+ if slave_list and slave in slave_list:
539
+ return True
540
+ return False
541
+
542
+
543
+ def check_disabled(master, builder, slave):
544
+ """Returns True if disabled, False if not disabled."""
545
+ builder_list = DISABLED_BUILDERS.get(master)
546
+ if builder_list and builder in builder_list:
547
+ return True
548
+ slave_list = DISABLED_SLAVES.get(master)
549
+ if slave_list and slave in slave_list:
550
+ return True
551
+ return False
552
+
553
+
554
+ def check_valid_host(master, builder, slave):
555
+ return (check_enabled(master, builder, slave)
556
+ and not check_disabled(master, builder, slave))
557
+
558
+
559
+ def maybe_ignore_revision(revision, buildspec):
560
+ """Handle builders that don't care what buildbot tells them to build.
561
+
562
+ This is especially the case with branch builders that build from buildspecs
563
+ and/or trigger off multiple repositories, where the --revision passed in has
564
+ nothing to do with the solution being built. Clearing the revision in this
565
+ case causes bot_update to use HEAD rather that trying to checkout an
566
+ inappropriate version of the solution.
567
+ """
568
+ if buildspec and buildspec.container == 'branches':
569
+ return []
570
+ return revision
571
+
572
+
573
+ def solutions_printer(solutions):
574
+ """Prints gclient solution to stdout."""
575
+ print 'Gclient Solutions'
576
+ print '================='
577
+ for solution in solutions:
578
+ name = solution.get('name')
579
+ url = solution.get('url')
580
+ print '%s (%s)' % (name, url)
581
+ if solution.get('deps_file'):
582
+ print ' Dependencies file is %s' % solution['deps_file']
583
+ if 'managed' in solution:
584
+ print ' Managed mode is %s' % ('ON' if solution['managed'] else 'OFF')
585
+ custom_vars = solution.get('custom_vars')
586
+ if custom_vars:
587
+ print ' Custom Variables:'
588
+ for var_name, var_value in sorted(custom_vars.iteritems()):
589
+ print ' %s = %s' % (var_name, var_value)
590
+ custom_deps = solution.get('custom_deps')
591
+ if 'custom_deps' in solution:
592
+ print ' Custom Dependencies:'
593
+ for deps_name, deps_value in sorted(custom_deps.iteritems()):
594
+ if deps_value:
595
+ print ' %s -> %s' % (deps_name, deps_value)
596
+ else:
597
+ print ' %s: Ignore' % deps_name
598
+ for k, v in solution.iteritems():
599
+ # Print out all the keys we don't know about.
600
+ if k in ['name', 'url', 'deps_file', 'custom_vars', 'custom_deps',
601
+ 'managed']:
602
+ continue
603
+ print ' %s is %s' % (k, v)
604
+ print
605
+
606
+
607
+ def solutions_to_git(input_solutions):
608
+ """Modifies urls in solutions to point at Git repos.
609
+
610
+ returns: (git solution, svn root of first solution) tuple.
611
+ """
612
+ assert input_solutions
613
+ solutions = copy.deepcopy(input_solutions)
614
+ first_solution = True
615
+ buildspec = None
616
+ for solution in solutions:
617
+ original_url = solution['url']
618
+ parsed_url = urlparse.urlparse(original_url)
619
+ parsed_path = parsed_url.path
620
+
621
+ # Rewrite SVN urls into Git urls.
622
+ buildspec_m = re.match(BUILDSPEC_RE, parsed_path)
623
+ if first_solution and buildspec_m:
624
+ solution['url'] = GIT_BUILDSPEC_PATH
625
+ buildspec = BUILDSPEC_TYPE(
626
+ container=buildspec_m.group(1),
627
+ version=buildspec_m.group(2),
628
+ )
629
+ solution['deps_file'] = path.join(buildspec.container, buildspec.version,
630
+ 'DEPS')
631
+ elif parsed_path in RECOGNIZED_PATHS:
632
+ solution['url'] = RECOGNIZED_PATHS[parsed_path]
633
+ solution['deps_file'] = '.DEPS.git'
634
+ elif parsed_url.scheme == 'https' and 'googlesource' in parsed_url.netloc:
635
+ pass
636
+ else:
637
+ print 'Warning: %s' % ('path %r not recognized' % parsed_path,)
638
+
639
+ # Strip out deps containing $$V8_REV$$, etc.
640
+ if 'custom_deps' in solution:
641
+ new_custom_deps = {}
642
+ for deps_name, deps_value in solution['custom_deps'].iteritems():
643
+ if deps_value and '$$' in deps_value:
644
+ print 'Dropping %s:%s from custom deps' % (deps_name, deps_value)
645
+ else:
646
+ new_custom_deps[deps_name] = deps_value
647
+ solution['custom_deps'] = new_custom_deps
648
+
649
+ if first_solution:
650
+ root = parsed_path
651
+ first_solution = False
652
+
653
+ solution['managed'] = False
654
+ # We don't want gclient to be using a safesync URL. Instead it should
655
+ # using the lkgr/lkcr branch/tags.
656
+ if 'safesync_url' in solution:
657
+ print 'Removing safesync url %s from %s' % (solution['safesync_url'],
658
+ parsed_path)
659
+ del solution['safesync_url']
660
+ return solutions, root, buildspec
661
+
662
+
663
+ def remove(target):
664
+ """Remove a target by moving it into build.dead."""
665
+ dead_folder = path.join(BUILDER_DIR, 'build.dead')
666
+ if not path.exists(dead_folder):
667
+ os.makedirs(dead_folder)
668
+ os.rename(target, path.join(dead_folder, uuid.uuid4().hex))
669
+
670
+
671
+ def ensure_no_checkout(dir_names, scm_dirname):
672
+ """Ensure that there is no undesired checkout under build/.
673
+
674
+ If there is an incorrect checkout under build/, then
675
+ move build/ to build.dead/
676
+ This function will check each directory in dir_names.
677
+
678
+ scm_dirname is expected to be either ['.svn', '.git']
679
+ """
680
+ assert scm_dirname in ['.svn', '.git', '*']
681
+ has_checkout = any(path.exists(path.join(os.getcwd(), dir_name, scm_dirname))
682
+ for dir_name in dir_names)
683
+
684
+ if has_checkout or scm_dirname == '*':
685
+ build_dir = os.getcwd()
686
+ prefix = ''
687
+ if scm_dirname != '*':
688
+ prefix = '%s detected in checkout, ' % scm_dirname
689
+
690
+ for filename in os.listdir(build_dir):
691
+ deletion_target = path.join(build_dir, filename)
692
+ print '%sdeleting %s...' % (prefix, deletion_target),
693
+ remove(deletion_target)
694
+ print 'done'
695
+
696
+
697
+ def gclient_configure(solutions, target_os, target_os_only, git_cache_dir):
698
+ """Should do the same thing as gclient --spec='...'."""
699
+ with codecs.open('.gclient', mode='w', encoding='utf-8') as f:
700
+ f.write(get_gclient_spec(
701
+ solutions, target_os, target_os_only, git_cache_dir))
702
+
703
+
704
+ def gclient_sync(with_branch_heads, shallow):
705
+ # We just need to allocate a filename.
706
+ fd, gclient_output_file = tempfile.mkstemp(suffix='.json')
707
+ os.close(fd)
708
+ gclient_bin = 'gclient.bat' if sys.platform.startswith('win') else 'gclient'
709
+ cmd = [gclient_bin, 'sync', '--verbose', '--reset', '--force',
710
+ '--ignore_locks', '--output-json', gclient_output_file,
711
+ '--nohooks', '--noprehooks', '--delete_unversioned_trees']
712
+ if with_branch_heads:
713
+ cmd += ['--with_branch_heads']
714
+ if shallow:
715
+ cmd += ['--shallow']
716
+
717
+ try:
718
+ call(*cmd, tries=1)
719
+ except SubprocessFailed as e:
720
+ # Throw a GclientSyncFailed exception so we can catch this independently.
721
+ raise GclientSyncFailed(e.message, e.code, e.output)
722
+ else:
723
+ with open(gclient_output_file) as f:
724
+ return json.load(f)
725
+ finally:
726
+ os.remove(gclient_output_file)
727
+
728
+
729
+ def gclient_runhooks(gyp_envs):
730
+ gclient_bin = 'gclient.bat' if sys.platform.startswith('win') else 'gclient'
731
+ env = dict([env_var.split('=', 1) for env_var in gyp_envs])
732
+ call(gclient_bin, 'runhooks', env=env)
733
+
734
+
735
+ def gclient_revinfo():
736
+ gclient_bin = 'gclient.bat' if sys.platform.startswith('win') else 'gclient'
737
+ return call(gclient_bin, 'revinfo', '-a') or ''
738
+
739
+
740
+ def create_manifest():
741
+ manifest = {}
742
+ output = gclient_revinfo()
743
+ for line in output.strip().splitlines():
744
+ match = REVINFO_RE.match(line.strip())
745
+ if match:
746
+ manifest[match.group(1)] = {
747
+ 'repository': match.group(2),
748
+ 'revision': match.group(3),
749
+ }
750
+ else:
751
+ print "WARNING: Couldn't match revinfo line:\n%s" % line
752
+ return manifest
753
+
754
+
755
+ def get_commit_message_footer_map(message):
756
+ """Returns: (dict) A dictionary of commit message footer entries.
757
+ """
758
+ footers = {}
759
+
760
+ # Extract the lines in the footer block.
761
+ lines = []
762
+ for line in message.strip().splitlines():
763
+ line = line.strip()
764
+ if len(line) == 0:
765
+ del lines[:]
766
+ continue
767
+ lines.append(line)
768
+
769
+ # Parse the footer
770
+ for line in lines:
771
+ m = COMMIT_FOOTER_ENTRY_RE.match(line)
772
+ if not m:
773
+ # If any single line isn't valid, the entire footer is invalid.
774
+ footers.clear()
775
+ return footers
776
+ footers[m.group(1)] = m.group(2).strip()
777
+ return footers
778
+
779
+
780
+ def get_commit_message_footer(message, key):
781
+ """Returns: (str/None) The footer value for 'key', or None if none was found.
782
+ """
783
+ return get_commit_message_footer_map(message).get(key)
784
+
785
+
786
+ def get_svn_rev(git_hash, dir_name):
787
+ log = git('log', '-1', git_hash, cwd=dir_name)
788
+ git_svn_id = get_commit_message_footer(log, GIT_SVN_ID_FOOTER_KEY)
789
+ if not git_svn_id:
790
+ return None
791
+ m = GIT_SVN_ID_RE.match(git_svn_id)
792
+ if not m:
793
+ return None
794
+ return int(m.group(2))
795
+
796
+
797
+ def get_git_hash(revision, branch, sln_dir):
798
+ """We want to search for the SVN revision on the git-svn branch.
799
+
800
+ Note that git will search backwards from origin/master.
801
+ """
802
+ match = "^%s: [^ ]*@%s " % (GIT_SVN_ID_FOOTER_KEY, revision)
803
+ ref = branch if branch.startswith('refs/') else 'origin/%s' % branch
804
+ cmd = ['log', '-E', '--grep', match, '--format=%H', '--max-count=1', ref]
805
+ result = git(*cmd, cwd=sln_dir).strip()
806
+ if result:
807
+ return result
808
+ raise SVNRevisionNotFound('We can\'t resolve svn r%s into a git hash in %s' %
809
+ (revision, sln_dir))
810
+
811
+
812
+ def _last_commit_for_file(filename, repo_base):
813
+ cmd = ['log', '--format=%H', '--max-count=1', '--', filename]
814
+ return git(*cmd, cwd=repo_base).strip()
815
+
816
+
817
+ def need_to_run_deps2git(repo_base, deps_file, deps_git_file):
818
+ """Checks to see if we need to run deps2git.
819
+
820
+ Returns True if there was a DEPS change after the last .DEPS.git update
821
+ or if DEPS has local modifications.
822
+ """
823
+ # See if DEPS is dirty
824
+ deps_file_status = git(
825
+ 'status', '--porcelain', deps_file, cwd=repo_base).strip()
826
+ if deps_file_status and deps_file_status.startswith('M '):
827
+ return True
828
+
829
+ last_known_deps_ref = _last_commit_for_file(deps_file, repo_base)
830
+ last_known_deps_git_ref = _last_commit_for_file(deps_git_file, repo_base)
831
+ merge_base_ref = git('merge-base', last_known_deps_ref,
832
+ last_known_deps_git_ref, cwd=repo_base).strip()
833
+
834
+ # If the merge base of the last DEPS and last .DEPS.git file is not
835
+ # equivilent to the hash of the last DEPS file, that means the DEPS file
836
+ # was committed after the last .DEPS.git file.
837
+ return last_known_deps_ref != merge_base_ref
838
+
839
+
840
+ def ensure_deps2git(solution, shallow, git_cache_dir):
841
+ repo_base = path.join(os.getcwd(), solution['name'])
842
+ deps_file = path.join(repo_base, 'DEPS')
843
+ deps_git_file = path.join(repo_base, '.DEPS.git')
844
+ if (not git('ls-files', 'DEPS', cwd=repo_base).strip() or
845
+ not git('ls-files', '.DEPS.git', cwd=repo_base).strip()):
846
+ return
847
+
848
+ print 'Checking if %s is newer than %s' % (deps_file, deps_git_file)
849
+ if not need_to_run_deps2git(repo_base, deps_file, deps_git_file):
850
+ return
851
+
852
+ print '===DEPS file modified, need to run deps2git==='
853
+ cmd = [sys.executable, DEPS2GIT_PATH,
854
+ '--workspace', os.getcwd(),
855
+ '--cache_dir', git_cache_dir,
856
+ '--deps', deps_file,
857
+ '--out', deps_git_file]
858
+ if 'chrome-internal.googlesource' in solution['url']:
859
+ cmd.extend(['--extra-rules', S2G_INTERNAL_PATH])
860
+ if shallow:
861
+ cmd.append('--shallow')
862
+ call(*cmd)
863
+
864
+
865
+ def emit_log_lines(name, lines):
866
+ for line in lines.splitlines():
867
+ print '@@@STEP_LOG_LINE@%s@%s@@@' % (name, line)
868
+ print '@@@STEP_LOG_END@%s@@@' % name
869
+
870
+
871
+ def emit_properties(properties):
872
+ for property_name, property_value in sorted(properties.items()):
873
+ print '@@@SET_BUILD_PROPERTY@%s@"%s"@@@' % (property_name, property_value)
874
+
875
+
876
+ # Derived from:
877
+ # http://code.activestate.com/recipes/577972-disk-usage/?in=user-4178764
878
+ def get_total_disk_space():
879
+ cwd = os.getcwd()
880
+ # Windows is the only platform that doesn't support os.statvfs, so
881
+ # we need to special case this.
882
+ if sys.platform.startswith('win'):
883
+ _, total, free = (ctypes.c_ulonglong(), ctypes.c_ulonglong(), \
884
+ ctypes.c_ulonglong())
885
+ if sys.version_info >= (3,) or isinstance(cwd, unicode):
886
+ fn = ctypes.windll.kernel32.GetDiskFreeSpaceExW
887
+ else:
888
+ fn = ctypes.windll.kernel32.GetDiskFreeSpaceExA
889
+ ret = fn(cwd, ctypes.byref(_), ctypes.byref(total), ctypes.byref(free))
890
+ if ret == 0:
891
+ # WinError() will fetch the last error code.
892
+ raise ctypes.WinError()
893
+ return (total.value, free.value)
894
+
895
+ else:
896
+ st = os.statvfs(cwd)
897
+ free = st.f_bavail * st.f_frsize
898
+ total = st.f_blocks * st.f_frsize
899
+ return (total, free)
900
+
901
+
902
+ def get_target_revision(folder_name, git_url, revisions):
903
+ normalized_name = folder_name.strip('/')
904
+ if normalized_name in revisions:
905
+ return revisions[normalized_name]
906
+ if git_url in revisions:
907
+ return revisions[git_url]
908
+ return None
909
+
910
+
911
+ def force_revision(folder_name, revision):
912
+ split_revision = revision.split(':', 1)
913
+ branch = 'master'
914
+ if len(split_revision) == 2:
915
+ # Support for "branch:revision" syntax.
916
+ branch, revision = split_revision
917
+
918
+ if revision and revision.upper() != 'HEAD':
919
+ if revision and revision.isdigit() and len(revision) < 40:
920
+ # rev_num is really a svn revision number, convert it into a git hash.
921
+ git_ref = get_git_hash(int(revision), branch, folder_name)
922
+ else:
923
+ # rev_num is actually a git hash or ref, we can just use it.
924
+ git_ref = revision
925
+ git('checkout', '--force', git_ref, cwd=folder_name)
926
+ else:
927
+ ref = branch if branch.startswith('refs/') else 'origin/%s' % branch
928
+ git('checkout', '--force', ref, cwd=folder_name)
929
+
930
+ def git_checkout(solutions, revisions, shallow, refs, git_cache_dir):
931
+ build_dir = os.getcwd()
932
+ # Before we do anything, break all git_cache locks.
933
+ if path.isdir(git_cache_dir):
934
+ git('cache', 'unlock', '-vv', '--force', '--all',
935
+ '--cache-dir', git_cache_dir)
936
+ for item in os.listdir(git_cache_dir):
937
+ filename = os.path.join(git_cache_dir, item)
938
+ if item.endswith('.lock'):
939
+ raise Exception('%s exists after cache unlock' % filename)
940
+ first_solution = True
941
+ for sln in solutions:
942
+ # This is so we can loop back and try again if we need to wait for the
943
+ # git mirrors to update from SVN.
944
+ done = False
945
+ tries_left = 60
946
+ while not done:
947
+ name = sln['name']
948
+ url = sln['url']
949
+ if url == CHROMIUM_SRC_URL or url + '.git' == CHROMIUM_SRC_URL:
950
+ # Experiments show there's little to be gained from
951
+ # a shallow clone of src.
952
+ shallow = False
953
+ sln_dir = path.join(build_dir, name)
954
+ s = ['--shallow'] if shallow else []
955
+ populate_cmd = (['cache', 'populate', '--ignore_locks', '-v',
956
+ '--cache-dir', git_cache_dir] + s + [url])
957
+ for ref in refs:
958
+ populate_cmd.extend(['--ref', ref])
959
+ git(*populate_cmd)
960
+ mirror_dir = git(
961
+ 'cache', 'exists', '--quiet',
962
+ '--cache-dir', git_cache_dir, url).strip()
963
+ clone_cmd = (
964
+ 'clone', '--no-checkout', '--local', '--shared', mirror_dir, sln_dir)
965
+
966
+ try:
967
+ if not path.isdir(sln_dir):
968
+ git(*clone_cmd)
969
+ else:
970
+ git('remote', 'set-url', 'origin', mirror_dir, cwd=sln_dir)
971
+ git('fetch', 'origin', cwd=sln_dir)
972
+ for ref in refs:
973
+ refspec = '%s:%s' % (ref, ref.lstrip('+'))
974
+ git('fetch', 'origin', refspec, cwd=sln_dir)
975
+
976
+ revision = get_target_revision(name, url, revisions) or 'HEAD'
977
+ force_revision(sln_dir, revision)
978
+ done = True
979
+ except SubprocessFailed as e:
980
+ # Exited abnormally, theres probably something wrong.
981
+ # Lets wipe the checkout and try again.
982
+ tries_left -= 1
983
+ if tries_left > 0:
984
+ print 'Something failed: %s.' % str(e)
985
+ print 'waiting 5 seconds and trying again...'
986
+ time.sleep(5)
987
+ else:
988
+ raise
989
+ remove(sln_dir)
990
+ except SVNRevisionNotFound:
991
+ tries_left -= 1
992
+ if tries_left > 0:
993
+ # If we don't have the correct revision, wait and try again.
994
+ print 'We can\'t find revision %s.' % revision
995
+ print 'The svn to git replicator is probably falling behind.'
996
+ print 'waiting 5 seconds and trying again...'
997
+ time.sleep(5)
998
+ else:
999
+ raise
1000
+
1001
+ git('clean', '-dff', cwd=sln_dir)
1002
+
1003
+ if first_solution:
1004
+ git_ref = git('log', '--format=%H', '--max-count=1',
1005
+ cwd=sln_dir).strip()
1006
+ first_solution = False
1007
+ return git_ref
1008
+
1009
+
1010
+ def _download(url):
1011
+ """Fetch url and return content, with retries for flake."""
1012
+ for attempt in xrange(ATTEMPTS):
1013
+ try:
1014
+ return urllib2.urlopen(url).read()
1015
+ except Exception:
1016
+ if attempt == ATTEMPTS - 1:
1017
+ raise
1018
+
1019
+
1020
+ def parse_diff(diff):
1021
+ """Takes a unified diff and returns a list of diffed files and their diffs.
1022
+
1023
+ The return format is a list of pairs of:
1024
+ (<filename>, <diff contents>)
1025
+ <diff contents> is inclusive of the diff line.
1026
+ """
1027
+ result = []
1028
+ current_diff = ''
1029
+ current_header = None
1030
+ for line in diff.splitlines():
1031
+ # "diff" is for git style patches, and "Index: " is for SVN style patches.
1032
+ if line.startswith('diff') or line.startswith('Index: '):
1033
+ if current_header:
1034
+ # If we are in a diff portion, then save the diff.
1035
+ result.append((current_header, '%s\n' % current_diff))
1036
+ git_header_match = re.match(r'diff (?:--git )?(\S+) (\S+)', line)
1037
+ svn_header_match = re.match(r'Index: (.*)', line)
1038
+
1039
+ if git_header_match:
1040
+ # First, see if its a git style header.
1041
+ from_file = git_header_match.group(1)
1042
+ to_file = git_header_match.group(2)
1043
+ if from_file != to_file and from_file.startswith('a/'):
1044
+ # Sometimes git prepends 'a/' and 'b/' in front of file paths.
1045
+ from_file = from_file[2:]
1046
+ current_header = from_file
1047
+
1048
+ elif svn_header_match:
1049
+ # Otherwise, check if its an SVN style header.
1050
+ current_header = svn_header_match.group(1)
1051
+
1052
+ else:
1053
+ # Otherwise... I'm not really sure what to do with this.
1054
+ raise InvalidDiff('Can\'t process header: %s\nFull diff:\n%s' %
1055
+ (line, diff))
1056
+
1057
+ current_diff = ''
1058
+ current_diff += '%s\n' % line
1059
+ if current_header:
1060
+ # We hit EOF, gotta save the last diff.
1061
+ result.append((current_header, current_diff))
1062
+ return result
1063
+
1064
+
1065
+ def get_svn_patch(patch_url):
1066
+ """Fetch patch from patch_url, return list of (filename, diff)"""
1067
+ svn_exe = 'svn.bat' if sys.platform.startswith('win') else 'svn'
1068
+ patch_data = call(svn_exe, 'cat', patch_url)
1069
+ return parse_diff(patch_data)
1070
+
1071
+
1072
+ def apply_svn_patch(patch_root, patches, whitelist=None, blacklist=None):
1073
+ """Expects a list of (filename, diff), applies it on top of patch_root."""
1074
+ if whitelist:
1075
+ patches = [(name, diff) for name, diff in patches if name in whitelist]
1076
+ elif blacklist:
1077
+ patches = [(name, diff) for name, diff in patches if name not in blacklist]
1078
+ diffs = [diff for _, diff in patches]
1079
+ patch = ''.join(diffs)
1080
+
1081
+ if patch:
1082
+ print '===Patching files==='
1083
+ for filename, _ in patches:
1084
+ print 'Patching %s' % filename
1085
+ try:
1086
+ call(PATCH_TOOL, '-p0', '--remove-empty-files', '--force', '--forward',
1087
+ stdin_data=patch, cwd=patch_root, tries=1)
1088
+ for filename, _ in patches:
1089
+ full_filename = path.abspath(path.join(patch_root, filename))
1090
+ git('add', full_filename, cwd=path.dirname(full_filename))
1091
+ except SubprocessFailed as e:
1092
+ raise PatchFailed(e.message, e.code, e.output)
1093
+
1094
+ def apply_rietveld_issue(issue, patchset, root, server, _rev_map, _revision,
1095
+ email_file, key_file, whitelist=None, blacklist=None):
1096
+ apply_issue_bin = ('apply_issue.bat' if sys.platform.startswith('win')
1097
+ else 'apply_issue')
1098
+ cmd = [apply_issue_bin,
1099
+ # The patch will be applied on top of this directory.
1100
+ '--root_dir', root,
1101
+ # Tell apply_issue how to fetch the patch.
1102
+ '--issue', issue,
1103
+ '--server', server,
1104
+ # Always run apply_issue.py, otherwise it would see update.flag
1105
+ # and then bail out.
1106
+ '--force',
1107
+ # Don't run gclient sync when it sees a DEPS change.
1108
+ '--ignore_deps',
1109
+ # TODO(tandrii): remove after http://crbug.com/537417 is resolved.
1110
+ # Temporary enable verbosity to see if Rietveld requests are actually
1111
+ # retried.
1112
+ '-v', '-v', # = logging.DEBUG level.
1113
+ ]
1114
+ # Use an oauth key file if specified.
1115
+ if email_file and key_file:
1116
+ cmd.extend(['--email-file', email_file, '--private-key-file', key_file])
1117
+ else:
1118
+ cmd.append('--no-auth')
1119
+
1120
+ if patchset:
1121
+ cmd.extend(['--patchset', patchset])
1122
+ if whitelist:
1123
+ for item in whitelist:
1124
+ cmd.extend(['--whitelist', item])
1125
+ elif blacklist:
1126
+ for item in blacklist:
1127
+ cmd.extend(['--blacklist', item])
1128
+
1129
+ # Only try once, since subsequent failures hide the real failure.
1130
+ try:
1131
+ call(*cmd, tries=1)
1132
+ except SubprocessFailed as e:
1133
+ raise PatchFailed(e.message, e.code, e.output)
1134
+
1135
+ def apply_gerrit_ref(gerrit_repo, gerrit_ref, root, gerrit_reset):
1136
+ gerrit_repo = gerrit_repo or 'origin'
1137
+ assert gerrit_ref
1138
+ try:
1139
+ base_rev = git('rev-parse', 'HEAD', cwd=root).strip()
1140
+ git('retry', 'fetch', gerrit_repo, gerrit_ref, cwd=root, tries=1)
1141
+ git('checkout', 'FETCH_HEAD', cwd=root)
1142
+ if gerrit_reset:
1143
+ git('reset', '--soft', base_rev, cwd=root)
1144
+ except SubprocessFailed as e:
1145
+ raise PatchFailed(e.message, e.code, e.output)
1146
+
1147
+ def check_flag(flag_file):
1148
+ """Returns True if the flag file is present."""
1149
+ return os.path.isfile(flag_file)
1150
+
1151
+
1152
+ def delete_flag(flag_file):
1153
+ """Remove bot update flag."""
1154
+ if os.path.isfile(flag_file):
1155
+ os.remove(flag_file)
1156
+
1157
+
1158
+ def emit_flag(flag_file):
1159
+ """Deposit a bot update flag on the system to tell gclient not to run."""
1160
+ print 'Emitting flag file at %s' % flag_file
1161
+ with open(flag_file, 'wb') as f:
1162
+ f.write('Success!')
1163
+
1164
+
1165
+ def get_commit_position_for_git_svn(url, revision):
1166
+ """Generates a commit position string for a 'git-svn' URL/revision.
1167
+
1168
+ If the 'git-svn' URL maps to a known project, we will construct a commit
1169
+ position branch value by applying substitution on the SVN URL.
1170
+ """
1171
+ # Identify the base URL so we can strip off trunk/branch name
1172
+ project_config = branch = None
1173
+ for _, project_config in GIT_SVN_PROJECT_MAP.iteritems():
1174
+ if url.startswith(project_config['svn_url']):
1175
+ branch = url[len(project_config['svn_url']):]
1176
+ break
1177
+
1178
+ if branch:
1179
+ # Strip any leading slashes
1180
+ branch = branch.lstrip('/')
1181
+
1182
+ # Try and map the branch
1183
+ for pattern, repl in project_config.get('branch_map', ()):
1184
+ nbranch, subn = re.subn(pattern, repl, branch, count=1)
1185
+ if subn:
1186
+ print 'INFO: Mapped SVN branch to Git branch [%s] => [%s]' % (
1187
+ branch, nbranch)
1188
+ branch = nbranch
1189
+ break
1190
+ else:
1191
+ # Use generic 'svn' branch
1192
+ print 'INFO: Could not resolve project for SVN URL %r' % (url,)
1193
+ branch = 'svn'
1194
+ return '%s@{#%s}' % (branch, revision)
1195
+
1196
+
1197
+ def get_commit_position(git_path, revision='HEAD'):
1198
+ """Dumps the 'git' log for a specific revision and parses out the commit
1199
+ position.
1200
+
1201
+ If a commit position metadata key is found, its value will be returned.
1202
+
1203
+ Otherwise, we will search for a 'git-svn' metadata entry. If one is found,
1204
+ we will compose a commit position from it, using its SVN revision value as
1205
+ the revision.
1206
+
1207
+ If the 'git-svn' URL maps to a known project, we will construct a commit
1208
+ position branch value by truncating the URL, mapping 'trunk' to
1209
+ "refs/heads/master". Otherwise, we will return the generic branch, 'svn'.
1210
+ """
1211
+ git_log = git('log', '--format=%B', '-n1', revision, cwd=git_path)
1212
+ footer_map = get_commit_message_footer_map(git_log)
1213
+
1214
+ # Search for commit position metadata
1215
+ value = (footer_map.get(COMMIT_POSITION_FOOTER_KEY) or
1216
+ footer_map.get(COMMIT_ORIGINAL_POSITION_FOOTER_KEY))
1217
+ if value:
1218
+ return value
1219
+
1220
+ # Compose a commit position from 'git-svn' metadata
1221
+ value = footer_map.get(GIT_SVN_ID_FOOTER_KEY)
1222
+ if value:
1223
+ m = GIT_SVN_ID_RE.match(value)
1224
+ if not m:
1225
+ raise ValueError("Invalid 'git-svn' value: [%s]" % (value,))
1226
+ return get_commit_position_for_git_svn(m.group(1), m.group(2))
1227
+ return None
1228
+
1229
+
1230
+ def parse_got_revision(gclient_output, got_revision_mapping, use_svn_revs):
1231
+ """Translate git gclient revision mapping to build properties.
1232
+
1233
+ If use_svn_revs is True, then translate git hashes in the revision mapping
1234
+ to svn revision numbers.
1235
+ """
1236
+ properties = {}
1237
+ solutions_output = {
1238
+ # Make sure path always ends with a single slash.
1239
+ '%s/' % path.rstrip('/') : solution_output for path, solution_output
1240
+ in gclient_output['solutions'].iteritems()
1241
+ }
1242
+ for dir_name, property_name in got_revision_mapping.iteritems():
1243
+ # Make sure dir_name always ends with a single slash.
1244
+ dir_name = '%s/' % dir_name.rstrip('/')
1245
+ if dir_name not in solutions_output:
1246
+ continue
1247
+ solution_output = solutions_output[dir_name]
1248
+ if solution_output.get('scm') is None:
1249
+ # This is an ignored DEPS, so the output got_revision should be 'None'.
1250
+ git_revision = revision = commit_position = None
1251
+ else:
1252
+ # Since we are using .DEPS.git, everything had better be git.
1253
+ assert solution_output.get('scm') == 'git'
1254
+ git_revision = git('rev-parse', 'HEAD', cwd=dir_name).strip()
1255
+ if use_svn_revs:
1256
+ revision = get_svn_rev(git_revision, dir_name)
1257
+ if not revision:
1258
+ revision = git_revision
1259
+ else:
1260
+ revision = git_revision
1261
+ commit_position = get_commit_position(dir_name)
1262
+
1263
+ properties[property_name] = revision
1264
+ if revision != git_revision:
1265
+ properties['%s_git' % property_name] = git_revision
1266
+ if commit_position:
1267
+ properties['%s_cp' % property_name] = commit_position
1268
+
1269
+ return properties
1270
+
1271
+
1272
+ def emit_json(out_file, did_run, gclient_output=None, **kwargs):
1273
+ """Write run information into a JSON file."""
1274
+ output = {}
1275
+ output.update(gclient_output if gclient_output else {})
1276
+ output.update({'did_run': did_run})
1277
+ output.update(kwargs)
1278
+ with open(out_file, 'wb') as f:
1279
+ f.write(json.dumps(output, sort_keys=True))
1280
+
1281
+
1282
+ def ensure_deps_revisions(deps_url_mapping, solutions, revisions):
1283
+ """Ensure correct DEPS revisions, ignores solutions."""
1284
+ for deps_name, deps_data in sorted(deps_url_mapping.items()):
1285
+ if deps_name.strip('/') in solutions:
1286
+ # This has already been forced to the correct solution by git_checkout().
1287
+ continue
1288
+ revision = get_target_revision(deps_name, deps_data.get('url', None),
1289
+ revisions)
1290
+ if not revision:
1291
+ continue
1292
+ # TODO(hinoka): Catch SVNRevisionNotFound error maybe?
1293
+ git('fetch', 'origin', cwd=deps_name)
1294
+ force_revision(deps_name, revision)
1295
+
1296
+
1297
+ def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only,
1298
+ patch_root, issue, patchset, patch_url, rietveld_server,
1299
+ gerrit_repo, gerrit_ref, revision_mapping,
1300
+ apply_issue_email_file, apply_issue_key_file, buildspec,
1301
+ gyp_env, shallow, runhooks, refs, git_cache_dir,
1302
+ gerrit_reset):
1303
+ # Get a checkout of each solution, without DEPS or hooks.
1304
+ # Calling git directly because there is no way to run Gclient without
1305
+ # invoking DEPS.
1306
+ print 'Fetching Git checkout'
1307
+
1308
+ git_ref = git_checkout(solutions, revisions, shallow, refs, git_cache_dir)
1309
+
1310
+ patches = None
1311
+ if patch_url:
1312
+ patches = get_svn_patch(patch_url)
1313
+
1314
+ already_patched = []
1315
+ patch_root = patch_root or ''
1316
+ for solution in solutions:
1317
+ if (patch_root == solution['name'] or
1318
+ solution['name'].startswith(patch_root + '/')):
1319
+ relative_root = solution['name'][len(patch_root) + 1:]
1320
+ target = '/'.join([relative_root, 'DEPS']).lstrip('/')
1321
+ if patches:
1322
+ apply_svn_patch(patch_root, patches, whitelist=[target])
1323
+ already_patched.append(target)
1324
+ elif issue:
1325
+ apply_rietveld_issue(issue, patchset, patch_root, rietveld_server,
1326
+ revision_mapping, git_ref, apply_issue_email_file,
1327
+ apply_issue_key_file, whitelist=[target])
1328
+ already_patched.append(target)
1329
+
1330
+ if not buildspec:
1331
+ # Run deps2git if there is a DEPS change after the last .DEPS.git commit.
1332
+ for solution in solutions:
1333
+ ensure_deps2git(solution, shallow, git_cache_dir)
1334
+
1335
+ # Ensure our build/ directory is set up with the correct .gclient file.
1336
+ gclient_configure(solutions, target_os, target_os_only, git_cache_dir)
1337
+
1338
+ # Let gclient do the DEPS syncing.
1339
+ # The branch-head refspec is a special case because its possible Chrome
1340
+ # src, which contains the branch-head refspecs, is DEPSed in.
1341
+ gclient_output = gclient_sync(buildspec or BRANCH_HEADS_REFSPEC in refs,
1342
+ shallow)
1343
+
1344
+ # Now that gclient_sync has finished, we should revert any .DEPS.git so that
1345
+ # presubmit doesn't complain about it being modified.
1346
+ if (not buildspec and
1347
+ git('ls-files', '.DEPS.git', cwd=first_sln).strip()):
1348
+ git('checkout', 'HEAD', '--', '.DEPS.git', cwd=first_sln)
1349
+
1350
+ if buildspec and runhooks:
1351
+ # Run gclient runhooks if we're on an official builder.
1352
+ # TODO(hinoka): Remove this when the official builders run their own
1353
+ # runhooks step.
1354
+ gclient_runhooks(gyp_env)
1355
+
1356
+ # Finally, ensure that all DEPS are pinned to the correct revision.
1357
+ dir_names = [sln['name'] for sln in solutions]
1358
+ ensure_deps_revisions(gclient_output.get('solutions', {}),
1359
+ dir_names, revisions)
1360
+ # Apply the rest of the patch here (sans DEPS)
1361
+ if patches:
1362
+ apply_svn_patch(patch_root, patches, blacklist=already_patched)
1363
+ elif issue:
1364
+ apply_rietveld_issue(issue, patchset, patch_root, rietveld_server,
1365
+ revision_mapping, git_ref, apply_issue_email_file,
1366
+ apply_issue_key_file, blacklist=already_patched)
1367
+ elif gerrit_ref:
1368
+ apply_gerrit_ref(gerrit_repo, gerrit_ref, patch_root, gerrit_reset)
1369
+
1370
+ # Reset the deps_file point in the solutions so that hooks get run properly.
1371
+ for sln in solutions:
1372
+ sln['deps_file'] = sln.get('deps_file', 'DEPS').replace('.DEPS.git', 'DEPS')
1373
+ gclient_configure(solutions, target_os, target_os_only, git_cache_dir)
1374
+
1375
+ return gclient_output
1376
+
1377
+
1378
+ def parse_revisions(revisions, root):
1379
+ """Turn a list of revision specs into a nice dictionary.
1380
+
1381
+ We will always return a dict with {root: something}. By default if root
1382
+ is unspecified, or if revisions is [], then revision will be assigned 'HEAD'
1383
+ """
1384
+ results = {root.strip('/'): 'HEAD'}
1385
+ expanded_revisions = []
1386
+ for revision in revisions:
1387
+ # Allow rev1,rev2,rev3 format.
1388
+ # TODO(hinoka): Delete this when webkit switches to recipes.
1389
+ expanded_revisions.extend(revision.split(','))
1390
+ for revision in expanded_revisions:
1391
+ split_revision = revision.split('@')
1392
+ if len(split_revision) == 1:
1393
+ # This is just a plain revision, set it as the revision for root.
1394
+ results[root] = split_revision[0]
1395
+ elif len(split_revision) == 2:
1396
+ # This is an alt_root@revision argument.
1397
+ current_root, current_rev = split_revision
1398
+
1399
+ # We want to normalize svn/git urls into .git urls.
1400
+ parsed_root = urlparse.urlparse(current_root)
1401
+ if parsed_root.scheme == 'svn':
1402
+ if parsed_root.path in RECOGNIZED_PATHS:
1403
+ normalized_root = RECOGNIZED_PATHS[parsed_root.path]
1404
+ else:
1405
+ print 'WARNING: SVN path %s not recognized, ignoring' % current_root
1406
+ continue
1407
+ elif parsed_root.scheme in ['http', 'https']:
1408
+ normalized_root = 'https://%s/%s' % (parsed_root.netloc,
1409
+ parsed_root.path)
1410
+ if not normalized_root.endswith('.git'):
1411
+ normalized_root = '%s.git' % normalized_root
1412
+ elif parsed_root.scheme:
1413
+ print 'WARNING: Unrecognized scheme %s, ignoring' % parsed_root.scheme
1414
+ continue
1415
+ else:
1416
+ # This is probably a local path.
1417
+ normalized_root = current_root.strip('/')
1418
+
1419
+ results[normalized_root] = current_rev
1420
+ else:
1421
+ print ('WARNING: %r is not recognized as a valid revision specification,'
1422
+ 'skipping' % revision)
1423
+ return results
1424
+
1425
+
1426
+ def parse_args():
1427
+ parse = optparse.OptionParser()
1428
+
1429
+ parse.add_option('--issue', help='Issue number to patch from.')
1430
+ parse.add_option('--patchset',
1431
+ help='Patchset from issue to patch from, if applicable.')
1432
+ parse.add_option('--apply_issue_email_file',
1433
+ help='--email-file option passthrough for apply_patch.py.')
1434
+ parse.add_option('--apply_issue_key_file',
1435
+ help='--private-key-file option passthrough for '
1436
+ 'apply_patch.py.')
1437
+ parse.add_option('--patch_url', help='Optional URL to SVN patch.')
1438
+ parse.add_option('--root', dest='patch_root',
1439
+ help='DEPRECATED: Use --patch_root.')
1440
+ parse.add_option('--patch_root', help='Directory to patch on top of.')
1441
+ parse.add_option('--rietveld_server',
1442
+ default='codereview.chromium.org',
1443
+ help='Rietveld server.')
1444
+ parse.add_option('--gerrit_repo',
1445
+ help='Gerrit repository to pull the ref from.')
1446
+ parse.add_option('--gerrit_ref', help='Gerrit ref to apply.')
1447
+ parse.add_option('--gerrit_no_reset', action='store_true',
1448
+ help='Bypass calling reset after applying a gerrit ref.')
1449
+ parse.add_option('--specs', help='Gcilent spec.')
1450
+ parse.add_option('--master', help='Master name.')
1451
+ parse.add_option('-f', '--force', action='store_true',
1452
+ help='Bypass check to see if we want to be run. '
1453
+ 'Should ONLY be used locally or by smart recipes.')
1454
+ parse.add_option('--revision_mapping',
1455
+ help='{"path/to/repo/": "property_name"}')
1456
+ parse.add_option('--revision_mapping_file',
1457
+ help=('Same as revision_mapping, except its a path to a json'
1458
+ ' file containing that format.'))
1459
+ parse.add_option('--revision', action='append', default=[],
1460
+ help='Revision to check out. Can be an SVN revision number, '
1461
+ 'git hash, or any form of git ref. Can prepend '
1462
+ 'root@<rev> to specify which repository, where root '
1463
+ 'is either a filesystem path, git https url, or '
1464
+ 'svn url. To specify Tip of Tree, set rev to HEAD.'
1465
+ 'To specify a git branch and an SVN rev, <rev> can be '
1466
+ 'set to <branch>:<revision>.')
1467
+ parse.add_option('--output_manifest', action='store_true',
1468
+ help=('Add manifest json to the json output.'))
1469
+ parse.add_option('--slave_name', default=socket.getfqdn().split('.')[0],
1470
+ help='Hostname of the current machine, '
1471
+ 'used for determining whether or not to activate.')
1472
+ parse.add_option('--builder_name', help='Name of the builder, '
1473
+ 'used for determining whether or not to activate.')
1474
+ parse.add_option('--build_dir', default=os.getcwd())
1475
+ parse.add_option('--flag_file', default=path.join(os.getcwd(),
1476
+ 'update.flag'))
1477
+ parse.add_option('--shallow', action='store_true',
1478
+ help='Use shallow clones for cache repositories.')
1479
+ parse.add_option('--gyp_env', action='append', default=[],
1480
+ help='Environment variables to pass into gclient runhooks.')
1481
+ parse.add_option('--clobber', action='store_true',
1482
+ help='Delete checkout first, always')
1483
+ parse.add_option('--bot_update_clobber', action='store_true', dest='clobber',
1484
+ help='(synonym for --clobber)')
1485
+ parse.add_option('-o', '--output_json',
1486
+ help='Output JSON information into a specified file')
1487
+ parse.add_option('--no_shallow', action='store_true',
1488
+ help='Bypass disk detection and never shallow clone. '
1489
+ 'Does not override the --shallow flag')
1490
+ parse.add_option('--no_runhooks', action='store_true',
1491
+ help='Do not run hooks on official builder.')
1492
+ parse.add_option('--refs', action='append',
1493
+ help='Also fetch this refspec for the main solution(s). '
1494
+ 'Eg. +refs/branch-heads/*')
1495
+ parse.add_option('--with_branch_heads', action='store_true',
1496
+ help='Always pass --with_branch_heads to gclient. This '
1497
+ 'does the same thing as --refs +refs/branch-heads/*')
1498
+ parse.add_option('--git-cache-dir', default=path.join(SLAVE_DIR, 'cache_dir'),
1499
+ help='Path to git cache directory.')
1500
+
1501
+
1502
+ options, args = parse.parse_args()
1503
+
1504
+ if not options.refs:
1505
+ options.refs = []
1506
+
1507
+ if options.with_branch_heads:
1508
+ options.refs.append(BRANCH_HEADS_REFSPEC)
1509
+ del options.with_branch_heads
1510
+
1511
+ try:
1512
+ if options.revision_mapping_file:
1513
+ if options.revision_mapping:
1514
+ print ('WARNING: Ignoring --revision_mapping: --revision_mapping_file '
1515
+ 'was set at the same time as --revision_mapping?')
1516
+ with open(options.revision_mapping_file, 'r') as f:
1517
+ options.revision_mapping = json.load(f)
1518
+ elif options.revision_mapping:
1519
+ options.revision_mapping = json.loads(options.revision_mapping)
1520
+ except Exception as e:
1521
+ print (
1522
+ 'WARNING: Caught execption while parsing revision_mapping*: %s'
1523
+ % (str(e),)
1524
+ )
1525
+
1526
+ # Because we print CACHE_DIR out into a .gclient file, and then later run
1527
+ # eval() on it, backslashes need to be escaped, otherwise "E:\b\build" gets
1528
+ # parsed as "E:[\x08][\x08]uild".
1529
+ if sys.platform.startswith('win'):
1530
+ options.git_cache_dir = options.git_cache_dir.replace('\\', '\\\\')
1531
+
1532
+ return options, args
1533
+
1534
+
1535
+ def prepare(options, git_slns, active):
1536
+ """Prepares the target folder before we checkout."""
1537
+ dir_names = [sln.get('name') for sln in git_slns if 'name' in sln]
1538
+ # If we're active now, but the flag file doesn't exist (we weren't active
1539
+ # last run) or vice versa, blow away all checkouts.
1540
+ if bool(active) != bool(check_flag(options.flag_file)):
1541
+ ensure_no_checkout(dir_names, '*')
1542
+ if options.output_json:
1543
+ # Make sure we tell recipes that we didn't run if the script exits here.
1544
+ emit_json(options.output_json, did_run=active)
1545
+ if active:
1546
+ if options.clobber:
1547
+ ensure_no_checkout(dir_names, '*')
1548
+ else:
1549
+ ensure_no_checkout(dir_names, '.svn')
1550
+ emit_flag(options.flag_file)
1551
+ else:
1552
+ delete_flag(options.flag_file)
1553
+ raise Inactive # This is caught in main() and we exit cleanly.
1554
+
1555
+ # Do a shallow checkout if the disk is less than 100GB.
1556
+ total_disk_space, free_disk_space = get_total_disk_space()
1557
+ total_disk_space_gb = int(total_disk_space / (1024 * 1024 * 1024))
1558
+ used_disk_space_gb = int((total_disk_space - free_disk_space)
1559
+ / (1024 * 1024 * 1024))
1560
+ percent_used = int(used_disk_space_gb * 100 / total_disk_space_gb)
1561
+ step_text = '[%dGB/%dGB used (%d%%)]' % (used_disk_space_gb,
1562
+ total_disk_space_gb,
1563
+ percent_used)
1564
+ if not options.output_json:
1565
+ print '@@@STEP_TEXT@%s@@@' % step_text
1566
+ if not options.shallow:
1567
+ options.shallow = (total_disk_space < SHALLOW_CLONE_THRESHOLD
1568
+ and not options.no_shallow)
1569
+
1570
+ # The first solution is where the primary DEPS file resides.
1571
+ first_sln = dir_names[0]
1572
+
1573
+ # Split all the revision specifications into a nice dict.
1574
+ print 'Revisions: %s' % options.revision
1575
+ revisions = parse_revisions(options.revision, first_sln)
1576
+ print 'Fetching Git checkout at %s@%s' % (first_sln, revisions[first_sln])
1577
+ return revisions, step_text
1578
+
1579
+
1580
+ def checkout(options, git_slns, specs, buildspec, master,
1581
+ svn_root, revisions, step_text):
1582
+ first_sln = git_slns[0]['name']
1583
+ dir_names = [sln.get('name') for sln in git_slns if 'name' in sln]
1584
+ try:
1585
+ # Outer try is for catching patch failures and exiting gracefully.
1586
+ # Inner try is for catching gclient failures and retrying gracefully.
1587
+ try:
1588
+ checkout_parameters = dict(
1589
+ # First, pass in the base of what we want to check out.
1590
+ solutions=git_slns,
1591
+ revisions=revisions,
1592
+ first_sln=first_sln,
1593
+
1594
+ # Also, target os variables for gclient.
1595
+ target_os=specs.get('target_os', []),
1596
+ target_os_only=specs.get('target_os_only', False),
1597
+
1598
+ # Then, pass in information about how to patch.
1599
+ patch_root=options.patch_root,
1600
+ issue=options.issue,
1601
+ patchset=options.patchset,
1602
+ patch_url=options.patch_url,
1603
+ rietveld_server=options.rietveld_server,
1604
+ gerrit_repo=options.gerrit_repo,
1605
+ gerrit_ref=options.gerrit_ref,
1606
+ revision_mapping=options.revision_mapping,
1607
+ apply_issue_email_file=options.apply_issue_email_file,
1608
+ apply_issue_key_file=options.apply_issue_key_file,
1609
+
1610
+ # For official builders.
1611
+ buildspec=buildspec,
1612
+ gyp_env=options.gyp_env,
1613
+ runhooks=not options.no_runhooks,
1614
+
1615
+ # Finally, extra configurations such as shallowness of the clone.
1616
+ shallow=options.shallow,
1617
+ refs=options.refs,
1618
+ git_cache_dir=options.git_cache_dir,
1619
+ gerrit_reset=not options.gerrit_no_reset)
1620
+ gclient_output = ensure_checkout(**checkout_parameters)
1621
+ except GclientSyncFailed:
1622
+ print 'We failed gclient sync, lets delete the checkout and retry.'
1623
+ ensure_no_checkout(dir_names, '*')
1624
+ gclient_output = ensure_checkout(**checkout_parameters)
1625
+ except PatchFailed as e:
1626
+ if options.output_json:
1627
+ # Tell recipes information such as root, got_revision, etc.
1628
+ emit_json(options.output_json,
1629
+ did_run=True,
1630
+ root=first_sln,
1631
+ log_lines=[('patch error', e.output),],
1632
+ patch_apply_return_code=e.code,
1633
+ patch_root=options.patch_root,
1634
+ patch_failure=True,
1635
+ step_text='%s PATCH FAILED' % step_text,
1636
+ fixed_revisions=revisions)
1637
+ else:
1638
+ # If we're not on recipes, tell annotator about our got_revisions.
1639
+ emit_log_lines('patch error', e.output)
1640
+ print '@@@STEP_TEXT@%s PATCH FAILED@@@' % step_text
1641
+ raise
1642
+
1643
+ # Revision is an svn revision, unless it's a git master.
1644
+ use_svn_rev = master not in GIT_MASTERS
1645
+
1646
+ # Take care of got_revisions outputs.
1647
+ revision_mapping = dict(GOT_REVISION_MAPPINGS.get(svn_root, {}))
1648
+ if options.revision_mapping:
1649
+ revision_mapping.update(options.revision_mapping)
1650
+
1651
+ # If the repo is not in the default GOT_REVISION_MAPPINGS and no
1652
+ # revision_mapping were specified on the command line then
1653
+ # default to setting 'got_revision' based on the first solution.
1654
+ if not revision_mapping:
1655
+ revision_mapping[first_sln] = 'got_revision'
1656
+
1657
+ got_revisions = parse_got_revision(gclient_output, revision_mapping,
1658
+ use_svn_rev)
1659
+
1660
+ if not got_revisions:
1661
+ # TODO(hinoka): We should probably bail out here, but in the interest
1662
+ # of giving mis-configured bots some time to get fixed use a dummy
1663
+ # revision here.
1664
+ got_revisions = { 'got_revision': 'BOT_UPDATE_NO_REV_FOUND' }
1665
+ #raise Exception('No got_revision(s) found in gclient output')
1666
+
1667
+ if options.output_json:
1668
+ manifest = create_manifest() if options.output_manifest else None
1669
+ # Tell recipes information such as root, got_revision, etc.
1670
+ emit_json(options.output_json,
1671
+ did_run=True,
1672
+ root=first_sln,
1673
+ patch_root=options.patch_root,
1674
+ step_text=step_text,
1675
+ fixed_revisions=revisions,
1676
+ properties=got_revisions,
1677
+ manifest=manifest)
1678
+ else:
1679
+ # If we're not on recipes, tell annotator about our got_revisions.
1680
+ emit_properties(got_revisions)
1681
+
1682
+
1683
+ def print_help_text(force, output_json, active, master, builder, slave):
1684
+ """Print helpful messages to tell devs whats going on."""
1685
+ if force and output_json:
1686
+ recipe_force = 'Forced on by recipes'
1687
+ elif active and output_json:
1688
+ recipe_force = 'Off by recipes, but forced on by bot update'
1689
+ elif not active and output_json:
1690
+ recipe_force = 'Forced off by recipes'
1691
+ else:
1692
+ recipe_force = 'N/A. Was not called by recipes'
1693
+
1694
+ print BOT_UPDATE_MESSAGE % {
1695
+ 'master': master or 'Not specified',
1696
+ 'builder': builder or 'Not specified',
1697
+ 'slave': slave or 'Not specified',
1698
+ 'recipe': recipe_force,
1699
+ 'CURRENT_DIR': CURRENT_DIR,
1700
+ 'BUILDER_DIR': BUILDER_DIR,
1701
+ 'SLAVE_DIR': SLAVE_DIR,
1702
+ 'THIS_DIR': THIS_DIR,
1703
+ 'SCRIPTS_DIR': SCRIPTS_DIR,
1704
+ 'BUILD_DIR': BUILD_DIR,
1705
+ 'ROOT_DIR': ROOT_DIR,
1706
+ 'DEPOT_TOOLS_DIR': DEPOT_TOOLS_DIR,
1707
+ },
1708
+ print ACTIVATED_MESSAGE if active else NOT_ACTIVATED_MESSAGE
1709
+
1710
+
1711
+ def main():
1712
+ # Get inputs.
1713
+ options, _ = parse_args()
1714
+ builder = options.builder_name
1715
+ slave = options.slave_name
1716
+ master = options.master
1717
+
1718
+ # Check if this script should activate or not.
1719
+ active = check_valid_host(master, builder, slave) or options.force or False
1720
+
1721
+ # Print a helpful message to tell developers whats going on with this step.
1722
+ print_help_text(
1723
+ options.force, options.output_json, active, master, builder, slave)
1724
+
1725
+ # Parse, munipulate, and print the gclient solutions.
1726
+ specs = {}
1727
+ exec(options.specs, specs)
1728
+ svn_solutions = specs.get('solutions', [])
1729
+ git_slns, svn_root, buildspec = solutions_to_git(svn_solutions)
1730
+ options.revision = maybe_ignore_revision(options.revision, buildspec)
1731
+
1732
+ solutions_printer(git_slns)
1733
+
1734
+ try:
1735
+ # Dun dun dun, the main part of bot_update.
1736
+ revisions, step_text = prepare(options, git_slns, active)
1737
+ checkout(options, git_slns, specs, buildspec, master, svn_root, revisions,
1738
+ step_text)
1739
+
1740
+ except Inactive:
1741
+ # Not active, should count as passing.
1742
+ pass
1743
+ except PatchFailed as e:
1744
+ emit_flag(options.flag_file)
1745
+ # Return a specific non-zero exit code for patch failure (because it is
1746
+ # a failure), but make it different than other failures to distinguish
1747
+ # between infra failures (independent from patch author), and patch
1748
+ # failures (that patch author can fix). However, PatchFailure due to
1749
+ # download patch failure is still an infra problem.
1750
+ if e.code == 3:
1751
+ # Patch download problem.
1752
+ return 87
1753
+ # Genuine patch problem.
1754
+ return 88
1755
+ except Exception:
1756
+ # Unexpected failure.
1757
+ emit_flag(options.flag_file)
1758
+ raise
1759
+ else:
1760
+ emit_flag(options.flag_file)
1761
+
1762
+
1763
+ if __name__ == '__main__':
1764
+ sys.exit(main())