activerecord-spanner-adapter 0.3.0 → 0.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (264) hide show
  1. checksums.yaml +5 -5
  2. data/.github/CODEOWNERS +7 -0
  3. data/.github/sync-repo-settings.yaml +16 -0
  4. data/.github/workflows/acceptance-tests-on-emulator.yaml +45 -0
  5. data/.github/workflows/acceptance-tests-on-production.yaml +36 -0
  6. data/.github/workflows/ci.yaml +33 -0
  7. data/.github/workflows/nightly-acceptance-tests-on-emulator.yaml +52 -0
  8. data/.github/workflows/nightly-acceptance-tests-on-production.yaml +35 -0
  9. data/.github/workflows/nightly-unit-tests.yaml +40 -0
  10. data/.github/workflows/release-please-label.yml +25 -0
  11. data/.github/workflows/release-please.yml +39 -0
  12. data/.github/workflows/rubocop.yaml +31 -0
  13. data/.gitignore +67 -5
  14. data/.kokoro/populate-secrets.sh +77 -0
  15. data/.kokoro/release.cfg +33 -0
  16. data/.kokoro/release.sh +15 -0
  17. data/.kokoro/trampoline_v2.sh +489 -0
  18. data/.rubocop.yml +46 -0
  19. data/.toys/release.rb +18 -0
  20. data/.trampolinerc +48 -0
  21. data/.yardopts +11 -0
  22. data/CHANGELOG.md +26 -0
  23. data/CODE_OF_CONDUCT.md +40 -0
  24. data/CONTRIBUTING.md +79 -0
  25. data/Gemfile +9 -4
  26. data/LICENSE +6 -6
  27. data/README.md +67 -30
  28. data/Rakefile +79 -3
  29. data/SECURITY.md +7 -0
  30. data/acceptance/cases/associations/has_many_associations_test.rb +119 -0
  31. data/acceptance/cases/associations/has_many_through_associations_test.rb +63 -0
  32. data/acceptance/cases/associations/has_one_associations_test.rb +79 -0
  33. data/acceptance/cases/associations/has_one_through_associations_test.rb +98 -0
  34. data/acceptance/cases/interleaved_associations/has_many_associations_using_interleaved_test.rb +211 -0
  35. data/acceptance/cases/migration/change_schema_test.rb +433 -0
  36. data/acceptance/cases/migration/change_table_test.rb +115 -0
  37. data/acceptance/cases/migration/column_attributes_test.rb +122 -0
  38. data/acceptance/cases/migration/column_positioning_test.rb +48 -0
  39. data/acceptance/cases/migration/columns_test.rb +201 -0
  40. data/acceptance/cases/migration/command_recorder_test.rb +406 -0
  41. data/acceptance/cases/migration/create_join_table_test.rb +216 -0
  42. data/acceptance/cases/migration/ddl_batching_test.rb +80 -0
  43. data/acceptance/cases/migration/foreign_key_test.rb +297 -0
  44. data/acceptance/cases/migration/index_test.rb +211 -0
  45. data/acceptance/cases/migration/references_foreign_key_test.rb +259 -0
  46. data/acceptance/cases/migration/references_index_test.rb +135 -0
  47. data/acceptance/cases/migration/references_statements_test.rb +166 -0
  48. data/acceptance/cases/migration/rename_column_test.rb +96 -0
  49. data/acceptance/cases/models/calculation_query_test.rb +128 -0
  50. data/acceptance/cases/models/generated_column_test.rb +126 -0
  51. data/acceptance/cases/models/mutation_test.rb +122 -0
  52. data/acceptance/cases/models/query_test.rb +147 -0
  53. data/acceptance/cases/sessions/session_not_found_test.rb +121 -0
  54. data/acceptance/cases/transactions/optimistic_locking_test.rb +141 -0
  55. data/acceptance/cases/transactions/read_only_transactions_test.rb +67 -0
  56. data/acceptance/cases/transactions/read_write_transactions_test.rb +248 -0
  57. data/acceptance/cases/type/all_types_test.rb +152 -0
  58. data/acceptance/cases/type/binary_test.rb +59 -0
  59. data/acceptance/cases/type/boolean_test.rb +31 -0
  60. data/acceptance/cases/type/date_test.rb +32 -0
  61. data/acceptance/cases/type/date_time_test.rb +30 -0
  62. data/acceptance/cases/type/float_test.rb +27 -0
  63. data/acceptance/cases/type/integer_test.rb +44 -0
  64. data/acceptance/cases/type/numeric_test.rb +27 -0
  65. data/acceptance/cases/type/string_test.rb +79 -0
  66. data/acceptance/cases/type/text_test.rb +30 -0
  67. data/acceptance/cases/type/time_test.rb +87 -0
  68. data/acceptance/models/account.rb +13 -0
  69. data/acceptance/models/address.rb +9 -0
  70. data/acceptance/models/album.rb +12 -0
  71. data/acceptance/models/all_types.rb +8 -0
  72. data/acceptance/models/author.rb +11 -0
  73. data/acceptance/models/club.rb +12 -0
  74. data/acceptance/models/comment.rb +9 -0
  75. data/acceptance/models/customer.rb +9 -0
  76. data/acceptance/models/department.rb +9 -0
  77. data/acceptance/models/firm.rb +10 -0
  78. data/acceptance/models/member.rb +13 -0
  79. data/acceptance/models/member_type.rb +9 -0
  80. data/acceptance/models/membership.rb +10 -0
  81. data/acceptance/models/organization.rb +9 -0
  82. data/acceptance/models/post.rb +10 -0
  83. data/acceptance/models/singer.rb +10 -0
  84. data/acceptance/models/track.rb +20 -0
  85. data/acceptance/models/transaction.rb +9 -0
  86. data/acceptance/schema/schema.rb +143 -0
  87. data/acceptance/test_helper.rb +260 -0
  88. data/activerecord-spanner-adapter.gemspec +32 -17
  89. data/assets/solidus-db.png +0 -0
  90. data/benchmarks/README.md +17 -0
  91. data/benchmarks/Rakefile +14 -0
  92. data/benchmarks/application.rb +308 -0
  93. data/benchmarks/config/database.yml +8 -0
  94. data/benchmarks/config/environment.rb +12 -0
  95. data/benchmarks/db/migrate/01_create_tables.rb +25 -0
  96. data/benchmarks/db/schema.rb +29 -0
  97. data/benchmarks/models/album.rb +9 -0
  98. data/benchmarks/models/singer.rb +9 -0
  99. data/bin/console +6 -7
  100. data/examples/rails/README.md +262 -0
  101. data/examples/snippets/README.md +29 -0
  102. data/examples/snippets/Rakefile +57 -0
  103. data/examples/snippets/array-data-type/README.md +45 -0
  104. data/examples/snippets/array-data-type/Rakefile +13 -0
  105. data/examples/snippets/array-data-type/application.rb +45 -0
  106. data/examples/snippets/array-data-type/config/database.yml +8 -0
  107. data/examples/snippets/array-data-type/db/migrate/01_create_tables.rb +24 -0
  108. data/examples/snippets/array-data-type/db/schema.rb +26 -0
  109. data/examples/snippets/array-data-type/db/seeds.rb +5 -0
  110. data/examples/snippets/array-data-type/models/entity_with_array_types.rb +18 -0
  111. data/examples/snippets/bin/create_emulator_instance.rb +18 -0
  112. data/examples/snippets/bulk-insert/README.md +21 -0
  113. data/examples/snippets/bulk-insert/Rakefile +13 -0
  114. data/examples/snippets/bulk-insert/application.rb +64 -0
  115. data/examples/snippets/bulk-insert/config/database.yml +8 -0
  116. data/examples/snippets/bulk-insert/db/migrate/01_create_tables.rb +21 -0
  117. data/examples/snippets/bulk-insert/db/schema.rb +26 -0
  118. data/examples/snippets/bulk-insert/db/seeds.rb +5 -0
  119. data/examples/snippets/bulk-insert/models/album.rb +9 -0
  120. data/examples/snippets/bulk-insert/models/singer.rb +9 -0
  121. data/examples/snippets/commit-timestamp/README.md +18 -0
  122. data/examples/snippets/commit-timestamp/Rakefile +13 -0
  123. data/examples/snippets/commit-timestamp/application.rb +53 -0
  124. data/examples/snippets/commit-timestamp/config/database.yml +8 -0
  125. data/examples/snippets/commit-timestamp/db/migrate/01_create_tables.rb +26 -0
  126. data/examples/snippets/commit-timestamp/db/schema.rb +29 -0
  127. data/examples/snippets/commit-timestamp/db/seeds.rb +5 -0
  128. data/examples/snippets/commit-timestamp/models/album.rb +9 -0
  129. data/examples/snippets/commit-timestamp/models/singer.rb +9 -0
  130. data/examples/snippets/config/environment.rb +21 -0
  131. data/examples/snippets/create-records/README.md +12 -0
  132. data/examples/snippets/create-records/Rakefile +13 -0
  133. data/examples/snippets/create-records/application.rb +42 -0
  134. data/examples/snippets/create-records/config/database.yml +8 -0
  135. data/examples/snippets/create-records/db/migrate/01_create_tables.rb +21 -0
  136. data/examples/snippets/create-records/db/schema.rb +26 -0
  137. data/examples/snippets/create-records/db/seeds.rb +5 -0
  138. data/examples/snippets/create-records/models/album.rb +9 -0
  139. data/examples/snippets/create-records/models/singer.rb +9 -0
  140. data/examples/snippets/date-data-type/README.md +19 -0
  141. data/examples/snippets/date-data-type/Rakefile +13 -0
  142. data/examples/snippets/date-data-type/application.rb +35 -0
  143. data/examples/snippets/date-data-type/config/database.yml +8 -0
  144. data/examples/snippets/date-data-type/db/migrate/01_create_tables.rb +20 -0
  145. data/examples/snippets/date-data-type/db/schema.rb +21 -0
  146. data/examples/snippets/date-data-type/db/seeds.rb +16 -0
  147. data/examples/snippets/date-data-type/models/singer.rb +8 -0
  148. data/examples/snippets/generated-column/README.md +41 -0
  149. data/examples/snippets/generated-column/Rakefile +13 -0
  150. data/examples/snippets/generated-column/application.rb +37 -0
  151. data/examples/snippets/generated-column/config/database.yml +8 -0
  152. data/examples/snippets/generated-column/db/migrate/01_create_tables.rb +23 -0
  153. data/examples/snippets/generated-column/db/schema.rb +21 -0
  154. data/examples/snippets/generated-column/db/seeds.rb +18 -0
  155. data/examples/snippets/generated-column/models/singer.rb +8 -0
  156. data/examples/snippets/interleaved-tables/README.md +152 -0
  157. data/examples/snippets/interleaved-tables/Rakefile +13 -0
  158. data/examples/snippets/interleaved-tables/application.rb +109 -0
  159. data/examples/snippets/interleaved-tables/config/database.yml +8 -0
  160. data/examples/snippets/interleaved-tables/db/migrate/01_create_tables.rb +44 -0
  161. data/examples/snippets/interleaved-tables/db/schema.rb +32 -0
  162. data/examples/snippets/interleaved-tables/db/seeds.rb +40 -0
  163. data/examples/snippets/interleaved-tables/models/album.rb +15 -0
  164. data/examples/snippets/interleaved-tables/models/singer.rb +20 -0
  165. data/examples/snippets/interleaved-tables/models/track.rb +25 -0
  166. data/examples/snippets/migrations/README.md +43 -0
  167. data/examples/snippets/migrations/Rakefile +13 -0
  168. data/examples/snippets/migrations/application.rb +26 -0
  169. data/examples/snippets/migrations/config/database.yml +8 -0
  170. data/examples/snippets/migrations/db/migrate/01_create_tables.rb +28 -0
  171. data/examples/snippets/migrations/db/schema.rb +33 -0
  172. data/examples/snippets/migrations/db/seeds.rb +5 -0
  173. data/examples/snippets/migrations/models/album.rb +10 -0
  174. data/examples/snippets/migrations/models/singer.rb +10 -0
  175. data/examples/snippets/migrations/models/track.rb +9 -0
  176. data/examples/snippets/mutations/README.md +34 -0
  177. data/examples/snippets/mutations/Rakefile +13 -0
  178. data/examples/snippets/mutations/application.rb +47 -0
  179. data/examples/snippets/mutations/config/database.yml +8 -0
  180. data/examples/snippets/mutations/db/migrate/01_create_tables.rb +22 -0
  181. data/examples/snippets/mutations/db/schema.rb +27 -0
  182. data/examples/snippets/mutations/db/seeds.rb +25 -0
  183. data/examples/snippets/mutations/models/album.rb +9 -0
  184. data/examples/snippets/mutations/models/singer.rb +9 -0
  185. data/examples/snippets/optimistic-locking/README.md +12 -0
  186. data/examples/snippets/optimistic-locking/Rakefile +13 -0
  187. data/examples/snippets/optimistic-locking/application.rb +48 -0
  188. data/examples/snippets/optimistic-locking/config/database.yml +8 -0
  189. data/examples/snippets/optimistic-locking/db/migrate/01_create_tables.rb +26 -0
  190. data/examples/snippets/optimistic-locking/db/schema.rb +29 -0
  191. data/examples/snippets/optimistic-locking/db/seeds.rb +25 -0
  192. data/examples/snippets/optimistic-locking/models/album.rb +9 -0
  193. data/examples/snippets/optimistic-locking/models/singer.rb +9 -0
  194. data/examples/snippets/quickstart/README.md +26 -0
  195. data/examples/snippets/quickstart/Rakefile +13 -0
  196. data/examples/snippets/quickstart/application.rb +51 -0
  197. data/examples/snippets/quickstart/config/database.yml +8 -0
  198. data/examples/snippets/quickstart/db/migrate/01_create_tables.rb +21 -0
  199. data/examples/snippets/quickstart/db/schema.rb +26 -0
  200. data/examples/snippets/quickstart/db/seeds.rb +24 -0
  201. data/examples/snippets/quickstart/models/album.rb +9 -0
  202. data/examples/snippets/quickstart/models/singer.rb +9 -0
  203. data/examples/snippets/read-only-transactions/README.md +13 -0
  204. data/examples/snippets/read-only-transactions/Rakefile +13 -0
  205. data/examples/snippets/read-only-transactions/application.rb +49 -0
  206. data/examples/snippets/read-only-transactions/config/database.yml +8 -0
  207. data/examples/snippets/read-only-transactions/db/migrate/01_create_tables.rb +21 -0
  208. data/examples/snippets/read-only-transactions/db/schema.rb +26 -0
  209. data/examples/snippets/read-only-transactions/db/seeds.rb +24 -0
  210. data/examples/snippets/read-only-transactions/models/album.rb +9 -0
  211. data/examples/snippets/read-only-transactions/models/singer.rb +9 -0
  212. data/examples/snippets/read-write-transactions/README.md +12 -0
  213. data/examples/snippets/read-write-transactions/Rakefile +13 -0
  214. data/examples/snippets/read-write-transactions/application.rb +39 -0
  215. data/examples/snippets/read-write-transactions/config/database.yml +8 -0
  216. data/examples/snippets/read-write-transactions/db/migrate/01_create_tables.rb +22 -0
  217. data/examples/snippets/read-write-transactions/db/schema.rb +27 -0
  218. data/examples/snippets/read-write-transactions/db/seeds.rb +25 -0
  219. data/examples/snippets/read-write-transactions/models/album.rb +9 -0
  220. data/examples/snippets/read-write-transactions/models/singer.rb +9 -0
  221. data/examples/snippets/timestamp-data-type/README.md +17 -0
  222. data/examples/snippets/timestamp-data-type/Rakefile +13 -0
  223. data/examples/snippets/timestamp-data-type/application.rb +42 -0
  224. data/examples/snippets/timestamp-data-type/config/database.yml +8 -0
  225. data/examples/snippets/timestamp-data-type/db/migrate/01_create_tables.rb +21 -0
  226. data/examples/snippets/timestamp-data-type/db/schema.rb +21 -0
  227. data/examples/snippets/timestamp-data-type/db/seeds.rb +6 -0
  228. data/examples/snippets/timestamp-data-type/models/meeting.rb +19 -0
  229. data/examples/solidus/README.md +172 -0
  230. data/lib/active_record/connection_adapters/spanner/database_statements.rb +224 -269
  231. data/lib/active_record/connection_adapters/spanner/quoting.rb +42 -50
  232. data/lib/active_record/connection_adapters/spanner/schema_cache.rb +43 -0
  233. data/lib/active_record/connection_adapters/spanner/schema_creation.rb +125 -9
  234. data/lib/active_record/connection_adapters/spanner/schema_definitions.rb +122 -0
  235. data/lib/active_record/connection_adapters/spanner/schema_dumper.rb +19 -0
  236. data/lib/active_record/connection_adapters/spanner/schema_statements.rb +553 -139
  237. data/lib/active_record/connection_adapters/spanner/type_metadata.rb +37 -0
  238. data/lib/active_record/connection_adapters/spanner_adapter.rb +182 -78
  239. data/lib/active_record/tasks/spanner_database_tasks.rb +74 -0
  240. data/lib/active_record/type/spanner/array.rb +32 -0
  241. data/lib/active_record/type/spanner/bytes.rb +26 -0
  242. data/lib/active_record/type/spanner/spanner_active_record_converter.rb +32 -0
  243. data/lib/active_record/type/spanner/time.rb +37 -0
  244. data/lib/activerecord-spanner-adapter.rb +23 -0
  245. data/lib/activerecord_spanner_adapter/base.rb +217 -0
  246. data/lib/activerecord_spanner_adapter/connection.rb +324 -0
  247. data/lib/activerecord_spanner_adapter/errors.rb +13 -0
  248. data/lib/activerecord_spanner_adapter/foreign_key.rb +29 -0
  249. data/lib/activerecord_spanner_adapter/index/column.rb +38 -0
  250. data/lib/activerecord_spanner_adapter/index.rb +80 -0
  251. data/lib/activerecord_spanner_adapter/information_schema.rb +261 -0
  252. data/lib/activerecord_spanner_adapter/primary_key.rb +31 -0
  253. data/lib/activerecord_spanner_adapter/table/column.rb +59 -0
  254. data/lib/activerecord_spanner_adapter/table.rb +61 -0
  255. data/lib/activerecord_spanner_adapter/transaction.rb +113 -0
  256. data/lib/activerecord_spanner_adapter/version.rb +9 -0
  257. data/lib/arel/visitors/spanner.rb +35 -0
  258. data/lib/spanner_client_ext.rb +82 -0
  259. data/renovate.json +5 -0
  260. metadata +387 -34
  261. data/.travis.yml +0 -5
  262. data/lib/active_record/connection_adapters/spanner/client.rb +0 -190
  263. data/lib/active_record/connection_adapters/spanner.rb +0 -10
  264. data/lib/activerecord-spanner-adapter/version.rb +0 -3
@@ -0,0 +1,77 @@
1
+ #!/bin/bash
2
+
3
+ # Copyright 2021 Google LLC
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+
17
+ # This file is called in the early stage of `trampoline_v2.sh` to
18
+ # populate secrets needed for the CI builds.
19
+
20
+ set -eo pipefail
21
+
22
+ function now { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n' ;}
23
+ function msg { println "$*" >&2 ;}
24
+ function println { printf '%s\n' "$(now) $*" ;}
25
+
26
+ # Populates requested secrets set in SECRET_MANAGER_KEYS
27
+
28
+ # In Kokoro CI builds, we use the service account attached to the
29
+ # Kokoro VM. This means we need to setup auth on other CI systems.
30
+ # For local run, we just use the gcloud command for retrieving the
31
+ # secrets.
32
+
33
+ if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then
34
+ GCLOUD_COMMANDS=(
35
+ "docker"
36
+ "run"
37
+ "--entrypoint=gcloud"
38
+ "--volume=${KOKORO_GFILE_DIR}:${KOKORO_GFILE_DIR}"
39
+ "gcr.io/google.com/cloudsdktool/cloud-sdk"
40
+ )
41
+ if [[ "${TRAMPOLINE_CI:-}" == "kokoro" ]]; then
42
+ SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager"
43
+ else
44
+ echo "Authentication for this CI system is not implemented yet."
45
+ exit 2
46
+ # TODO: Determine appropriate SECRET_LOCATION and the GCLOUD_COMMANDS.
47
+ fi
48
+ else
49
+ # For local run, use /dev/shm or temporary directory for
50
+ # KOKORO_GFILE_DIR.
51
+ if [[ -d "/dev/shm" ]]; then
52
+ export KOKORO_GFILE_DIR=/dev/shm
53
+ else
54
+ export KOKORO_GFILE_DIR=$(mktemp -d -t ci-XXXXXXXX)
55
+ fi
56
+ SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager"
57
+ GCLOUD_COMMANDS=("gcloud")
58
+ fi
59
+
60
+ msg "Creating folder on disk for secrets: ${SECRET_LOCATION}"
61
+ mkdir -p ${SECRET_LOCATION}
62
+
63
+ for key in $(echo ${SECRET_MANAGER_KEYS} | sed "s/,/ /g")
64
+ do
65
+ msg "Retrieving secret ${key}"
66
+ "${GCLOUD_COMMANDS[@]}" \
67
+ secrets versions access latest \
68
+ --project cloud-devrel-kokoro-resources \
69
+ --secret $key > \
70
+ "$SECRET_LOCATION/$key"
71
+ if [[ $? == 0 ]]; then
72
+ msg "Secret written to ${SECRET_LOCATION}/${key}"
73
+ else
74
+ msg "Error retrieving secret ${key}"
75
+ exit 2
76
+ fi
77
+ done
@@ -0,0 +1,33 @@
1
+ # Format: //devtools/kokoro/config/proto/build.proto
2
+
3
+ # Build logs will be here
4
+ action {
5
+ define_artifacts {
6
+ regex: "**/*sponge_log.xml"
7
+ }
8
+ }
9
+
10
+ # Download resources for system tests (service account key, etc.)
11
+ gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/google-cloud-ruby"
12
+
13
+ # Download trampoline resources.
14
+ gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
15
+
16
+ # Use the trampoline script to run in docker.
17
+ build_file: "ruby-spanner-activerecord/.kokoro/trampoline_v2.sh"
18
+
19
+ # Configure the docker image for kokoro-trampoline.
20
+ env_vars: {
21
+ key: "TRAMPOLINE_IMAGE"
22
+ value: "gcr.io/cloud-devrel-kokoro-resources/yoshi-ruby/release"
23
+ }
24
+
25
+ env_vars: {
26
+ key: "TRAMPOLINE_BUILD_FILE"
27
+ value: ".kokoro/release.sh"
28
+ }
29
+
30
+ env_vars: {
31
+ key: "SECRET_MANAGER_KEYS"
32
+ value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem,docuploader_service_account"
33
+ }
@@ -0,0 +1,15 @@
1
+ #!/bin/bash
2
+
3
+ set -eo pipefail
4
+
5
+ # Install gems in the user directory because the default install directory
6
+ # is in a read-only location.
7
+ export GEM_HOME=$HOME/.gem
8
+ export PATH=$GEM_HOME/bin:$PATH
9
+
10
+ python3 -m pip install git+https://github.com/googleapis/releasetool
11
+ python3 -m pip install gcp-docuploader
12
+ gem install --no-document toys
13
+
14
+ python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script
15
+ toys release perform -v --enable-docs < /dev/null
@@ -0,0 +1,489 @@
1
+ #!/usr/bin/env bash
2
+ # Copyright 2020 Google LLC
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ # trampoline_v2.sh
17
+ #
18
+ # If you want to make a change to this file, consider doing so at:
19
+ # https://github.com/googlecloudplatform/docker-ci-helper
20
+ #
21
+ # This script is for running CI builds. For Kokoro builds, we
22
+ # set this script to `build_file` field in the Kokoro configuration.
23
+
24
+ # This script does 3 things.
25
+ #
26
+ # 1. Prepare the Docker image for the test
27
+ # 2. Run the Docker with appropriate flags to run the test
28
+ # 3. Upload the newly built Docker image
29
+ #
30
+ # in a way that is somewhat compatible with trampoline_v1.
31
+ #
32
+ # These environment variables are required:
33
+ # TRAMPOLINE_IMAGE: The docker image to use.
34
+ # TRAMPOLINE_DOCKERFILE: The location of the Dockerfile.
35
+ #
36
+ # You can optionally change these environment variables:
37
+ # TRAMPOLINE_IMAGE_UPLOAD:
38
+ # (true|false): Whether to upload the Docker image after the
39
+ # successful builds.
40
+ # TRAMPOLINE_BUILD_FILE: The script to run in the docker container.
41
+ # TRAMPOLINE_WORKSPACE: The workspace path in the docker container.
42
+ # Defaults to /workspace.
43
+ # Potentially there are some repo specific envvars in .trampolinerc in
44
+ # the project root.
45
+ #
46
+ # Here is an example for running this script.
47
+ # TRAMPOLINE_IMAGE=gcr.io/cloud-devrel-kokoro-resources/node:10-user \
48
+ # TRAMPOLINE_BUILD_FILE=.kokoro/system-test.sh \
49
+ # .kokoro/trampoline_v2.sh
50
+
51
+ set -euo pipefail
52
+
53
+ TRAMPOLINE_VERSION="2.0.10"
54
+
55
+ if command -v tput >/dev/null && [[ -n "${TERM:-}" ]]; then
56
+ readonly IO_COLOR_RED="$(tput setaf 1)"
57
+ readonly IO_COLOR_GREEN="$(tput setaf 2)"
58
+ readonly IO_COLOR_YELLOW="$(tput setaf 3)"
59
+ readonly IO_COLOR_RESET="$(tput sgr0)"
60
+ else
61
+ readonly IO_COLOR_RED=""
62
+ readonly IO_COLOR_GREEN=""
63
+ readonly IO_COLOR_YELLOW=""
64
+ readonly IO_COLOR_RESET=""
65
+ fi
66
+
67
+ function function_exists {
68
+ [ $(LC_ALL=C type -t $1)"" == "function" ]
69
+ }
70
+
71
+ # Logs a message using the given color. The first argument must be one
72
+ # of the IO_COLOR_* variables defined above, such as
73
+ # "${IO_COLOR_YELLOW}". The remaining arguments will be logged in the
74
+ # given color. The log message will also have an RFC-3339 timestamp
75
+ # prepended (in UTC). You can disable the color output by setting
76
+ # TERM=vt100.
77
+ function log_impl() {
78
+ local color="$1"
79
+ shift
80
+ local timestamp="$(date -u "+%Y-%m-%dT%H:%M:%SZ")"
81
+ echo "================================================================"
82
+ echo "${color}${timestamp}:" "$@" "${IO_COLOR_RESET}"
83
+ echo "================================================================"
84
+ }
85
+
86
+ # Logs the given message with normal coloring and a timestamp.
87
+ function log() {
88
+ log_impl "${IO_COLOR_RESET}" "$@"
89
+ }
90
+
91
+ # Logs the given message in green with a timestamp.
92
+ function log_green() {
93
+ log_impl "${IO_COLOR_GREEN}" "$@"
94
+ }
95
+
96
+ # Logs the given message in yellow with a timestamp.
97
+ function log_yellow() {
98
+ log_impl "${IO_COLOR_YELLOW}" "$@"
99
+ }
100
+
101
+ # Logs the given message in red with a timestamp.
102
+ function log_red() {
103
+ log_impl "${IO_COLOR_RED}" "$@"
104
+ }
105
+
106
+ readonly tmpdir=$(mktemp -d -t ci-XXXXXXXX)
107
+ readonly tmphome="${tmpdir}/h"
108
+ mkdir -p "${tmphome}"
109
+
110
+ function cleanup() {
111
+ rm -rf "${tmpdir}"
112
+ }
113
+ trap cleanup EXIT
114
+
115
+ RUNNING_IN_CI="${RUNNING_IN_CI:-false}"
116
+
117
+ # The workspace in the container, defaults to /workspace.
118
+ TRAMPOLINE_WORKSPACE="${TRAMPOLINE_WORKSPACE:-/workspace}"
119
+
120
+ pass_down_envvars=(
121
+ # TRAMPOLINE_V2 variables.
122
+ # Tells scripts whether they are running as part of CI or not.
123
+ "RUNNING_IN_CI"
124
+ # Indicates which CI system we're in.
125
+ "TRAMPOLINE_CI"
126
+ # Indicates the version of the script.
127
+ "TRAMPOLINE_VERSION"
128
+ )
129
+
130
+ log_yellow "Building with Trampoline ${TRAMPOLINE_VERSION}"
131
+
132
+ # Detect which CI systems we're in. If we're in any of the CI systems
133
+ # we support, `RUNNING_IN_CI` will be true and `TRAMPOLINE_CI` will be
134
+ # the name of the CI system. Both envvars will be passing down to the
135
+ # container for telling which CI system we're in.
136
+ if [[ -n "${KOKORO_BUILD_ID:-}" ]]; then
137
+ # descriptive env var for indicating it's on CI.
138
+ RUNNING_IN_CI="true"
139
+ TRAMPOLINE_CI="kokoro"
140
+ if [[ "${TRAMPOLINE_USE_LEGACY_SERVICE_ACCOUNT:-}" == "true" ]]; then
141
+ if [[ ! -f "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json" ]]; then
142
+ log_red "${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json does not exist. Did you forget to mount cloud-devrel-kokoro-resources/trampoline? Aborting."
143
+ exit 1
144
+ fi
145
+ # This service account will be activated later.
146
+ TRAMPOLINE_SERVICE_ACCOUNT="${KOKORO_GFILE_DIR}/kokoro-trampoline.service-account.json"
147
+ else
148
+ if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then
149
+ gcloud auth list
150
+ fi
151
+ log_yellow "Configuring Container Registry access"
152
+ gcloud auth configure-docker --quiet
153
+ fi
154
+ pass_down_envvars+=(
155
+ # KOKORO dynamic variables.
156
+ "KOKORO_BUILD_NUMBER"
157
+ "KOKORO_BUILD_ID"
158
+ "KOKORO_JOB_NAME"
159
+ "KOKORO_GIT_COMMIT"
160
+ "KOKORO_GITHUB_COMMIT"
161
+ "KOKORO_GITHUB_PULL_REQUEST_NUMBER"
162
+ "KOKORO_GITHUB_PULL_REQUEST_COMMIT"
163
+ # For Flaky Bot
164
+ "KOKORO_GITHUB_COMMIT_URL"
165
+ "KOKORO_GITHUB_PULL_REQUEST_URL"
166
+ "KOKORO_BUILD_ARTIFACTS_SUBDIR"
167
+ )
168
+ elif [[ "${TRAVIS:-}" == "true" ]]; then
169
+ RUNNING_IN_CI="true"
170
+ TRAMPOLINE_CI="travis"
171
+ pass_down_envvars+=(
172
+ "TRAVIS_BRANCH"
173
+ "TRAVIS_BUILD_ID"
174
+ "TRAVIS_BUILD_NUMBER"
175
+ "TRAVIS_BUILD_WEB_URL"
176
+ "TRAVIS_COMMIT"
177
+ "TRAVIS_COMMIT_MESSAGE"
178
+ "TRAVIS_COMMIT_RANGE"
179
+ "TRAVIS_JOB_NAME"
180
+ "TRAVIS_JOB_NUMBER"
181
+ "TRAVIS_JOB_WEB_URL"
182
+ "TRAVIS_PULL_REQUEST"
183
+ "TRAVIS_PULL_REQUEST_BRANCH"
184
+ "TRAVIS_PULL_REQUEST_SHA"
185
+ "TRAVIS_PULL_REQUEST_SLUG"
186
+ "TRAVIS_REPO_SLUG"
187
+ "TRAVIS_SECURE_ENV_VARS"
188
+ "TRAVIS_TAG"
189
+ )
190
+ elif [[ -n "${GITHUB_RUN_ID:-}" ]]; then
191
+ RUNNING_IN_CI="true"
192
+ TRAMPOLINE_CI="github-workflow"
193
+ pass_down_envvars+=(
194
+ "GITHUB_WORKFLOW"
195
+ "GITHUB_RUN_ID"
196
+ "GITHUB_RUN_NUMBER"
197
+ "GITHUB_ACTION"
198
+ "GITHUB_ACTIONS"
199
+ "GITHUB_ACTOR"
200
+ "GITHUB_REPOSITORY"
201
+ "GITHUB_EVENT_NAME"
202
+ "GITHUB_EVENT_PATH"
203
+ "GITHUB_SHA"
204
+ "GITHUB_REF"
205
+ "GITHUB_HEAD_REF"
206
+ "GITHUB_BASE_REF"
207
+ )
208
+ elif [[ "${CIRCLECI:-}" == "true" ]]; then
209
+ RUNNING_IN_CI="true"
210
+ TRAMPOLINE_CI="circleci"
211
+ pass_down_envvars+=(
212
+ "CIRCLE_BRANCH"
213
+ "CIRCLE_BUILD_NUM"
214
+ "CIRCLE_BUILD_URL"
215
+ "CIRCLE_COMPARE_URL"
216
+ "CIRCLE_JOB"
217
+ "CIRCLE_NODE_INDEX"
218
+ "CIRCLE_NODE_TOTAL"
219
+ "CIRCLE_PREVIOUS_BUILD_NUM"
220
+ "CIRCLE_PROJECT_REPONAME"
221
+ "CIRCLE_PROJECT_USERNAME"
222
+ "CIRCLE_REPOSITORY_URL"
223
+ "CIRCLE_SHA1"
224
+ "CIRCLE_STAGE"
225
+ "CIRCLE_USERNAME"
226
+ "CIRCLE_WORKFLOW_ID"
227
+ "CIRCLE_WORKFLOW_JOB_ID"
228
+ "CIRCLE_WORKFLOW_UPSTREAM_JOB_IDS"
229
+ "CIRCLE_WORKFLOW_WORKSPACE_ID"
230
+ )
231
+ fi
232
+
233
+ # Configure the service account for pulling the docker image.
234
+ function repo_root() {
235
+ local dir="$1"
236
+ while [[ ! -d "${dir}/.git" ]]; do
237
+ dir="$(dirname "$dir")"
238
+ done
239
+ echo "${dir}"
240
+ }
241
+
242
+ # Detect the project root. In CI builds, we assume the script is in
243
+ # the git tree and traverse from there, otherwise, traverse from `pwd`
244
+ # to find `.git` directory.
245
+ if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then
246
+ PROGRAM_PATH="$(realpath "$0")"
247
+ PROGRAM_DIR="$(dirname "${PROGRAM_PATH}")"
248
+ PROJECT_ROOT="$(repo_root "${PROGRAM_DIR}")"
249
+ else
250
+ PROJECT_ROOT="$(repo_root $(pwd))"
251
+ fi
252
+
253
+ log_yellow "Changing to the project root: ${PROJECT_ROOT}."
254
+ cd "${PROJECT_ROOT}"
255
+
256
+ # To support relative path for `TRAMPOLINE_SERVICE_ACCOUNT`, we need
257
+ # to use this environment variable in `PROJECT_ROOT`.
258
+ if [[ -n "${TRAMPOLINE_SERVICE_ACCOUNT:-}" ]]; then
259
+
260
+ mkdir -p "${tmpdir}/gcloud"
261
+ gcloud_config_dir="${tmpdir}/gcloud"
262
+
263
+ log_yellow "Using isolated gcloud config: ${gcloud_config_dir}."
264
+ export CLOUDSDK_CONFIG="${gcloud_config_dir}"
265
+
266
+ log_yellow "Using ${TRAMPOLINE_SERVICE_ACCOUNT} for authentication."
267
+ gcloud auth activate-service-account \
268
+ --key-file "${TRAMPOLINE_SERVICE_ACCOUNT}"
269
+ log_yellow "Configuring Container Registry access"
270
+ gcloud auth configure-docker --quiet
271
+ fi
272
+
273
+ required_envvars=(
274
+ # The basic trampoline configurations.
275
+ "TRAMPOLINE_IMAGE"
276
+ "TRAMPOLINE_BUILD_FILE"
277
+ )
278
+
279
+ if [[ -f "${PROJECT_ROOT}/.trampolinerc" ]]; then
280
+ source "${PROJECT_ROOT}/.trampolinerc"
281
+ fi
282
+
283
+ log_yellow "Checking environment variables."
284
+ for e in "${required_envvars[@]}"
285
+ do
286
+ if [[ -z "${!e:-}" ]]; then
287
+ log "Missing ${e} env var. Aborting."
288
+ exit 1
289
+ fi
290
+ done
291
+
292
+ # We want to support legacy style TRAMPOLINE_BUILD_FILE used with V1
293
+ # script: e.g. "github/repo-name/.kokoro/run_tests.sh"
294
+ TRAMPOLINE_BUILD_FILE="${TRAMPOLINE_BUILD_FILE#github/*/}"
295
+ log_yellow "Using TRAMPOLINE_BUILD_FILE: ${TRAMPOLINE_BUILD_FILE}"
296
+
297
+ # ignore error on docker operations and test execution
298
+ set +e
299
+
300
+ log_yellow "Preparing Docker image."
301
+ # We only download the docker image in CI builds.
302
+ if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then
303
+ # Download the docker image specified by `TRAMPOLINE_IMAGE`
304
+
305
+ # We may want to add --max-concurrent-downloads flag.
306
+
307
+ log_yellow "Start pulling the Docker image: ${TRAMPOLINE_IMAGE}."
308
+ if docker pull "${TRAMPOLINE_IMAGE}"; then
309
+ log_green "Finished pulling the Docker image: ${TRAMPOLINE_IMAGE}."
310
+ has_image="true"
311
+ else
312
+ log_red "Failed pulling the Docker image: ${TRAMPOLINE_IMAGE}."
313
+ has_image="false"
314
+ fi
315
+ else
316
+ # For local run, check if we have the image.
317
+ if docker images "${TRAMPOLINE_IMAGE}" | grep "${TRAMPOLINE_IMAGE%:*}"; then
318
+ has_image="true"
319
+ else
320
+ has_image="false"
321
+ fi
322
+ fi
323
+
324
+
325
+ # The default user for a Docker container has uid 0 (root). To avoid
326
+ # creating root-owned files in the build directory we tell docker to
327
+ # use the current user ID.
328
+ user_uid="$(id -u)"
329
+ user_gid="$(id -g)"
330
+ user_name="$(id -un)"
331
+
332
+ # To allow docker in docker, we add the user to the docker group in
333
+ # the host os.
334
+ docker_gid=$(cut -d: -f3 < <(getent group docker))
335
+
336
+ update_cache="false"
337
+ if [[ "${TRAMPOLINE_DOCKERFILE:-none}" != "none" ]]; then
338
+ # Build the Docker image from the source.
339
+ context_dir=$(dirname "${TRAMPOLINE_DOCKERFILE}")
340
+ docker_build_flags=(
341
+ "-f" "${TRAMPOLINE_DOCKERFILE}"
342
+ "-t" "${TRAMPOLINE_IMAGE}"
343
+ "--build-arg" "UID=${user_uid}"
344
+ "--build-arg" "USERNAME=${user_name}"
345
+ )
346
+ if [[ "${has_image}" == "true" ]]; then
347
+ docker_build_flags+=("--cache-from" "${TRAMPOLINE_IMAGE}")
348
+ fi
349
+
350
+ log_yellow "Start building the docker image."
351
+ if [[ "${TRAMPOLINE_VERBOSE:-false}" == "true" ]]; then
352
+ echo "docker build" "${docker_build_flags[@]}" "${context_dir}"
353
+ fi
354
+
355
+ # ON CI systems, we want to suppress docker build logs, only
356
+ # output the logs when it fails.
357
+ if [[ "${RUNNING_IN_CI:-}" == "true" ]]; then
358
+ if docker build "${docker_build_flags[@]}" "${context_dir}" \
359
+ > "${tmpdir}/docker_build.log" 2>&1; then
360
+ if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then
361
+ cat "${tmpdir}/docker_build.log"
362
+ fi
363
+
364
+ log_green "Finished building the docker image."
365
+ update_cache="true"
366
+ else
367
+ log_red "Failed to build the Docker image, aborting."
368
+ log_yellow "Dumping the build logs:"
369
+ cat "${tmpdir}/docker_build.log"
370
+ exit 1
371
+ fi
372
+ else
373
+ if docker build "${docker_build_flags[@]}" "${context_dir}"; then
374
+ log_green "Finished building the docker image."
375
+ update_cache="true"
376
+ else
377
+ log_red "Failed to build the Docker image, aborting."
378
+ exit 1
379
+ fi
380
+ fi
381
+ else
382
+ if [[ "${has_image}" != "true" ]]; then
383
+ log_red "We do not have ${TRAMPOLINE_IMAGE} locally, aborting."
384
+ exit 1
385
+ fi
386
+ fi
387
+
388
+ # We use an array for the flags so they are easier to document.
389
+ docker_flags=(
390
+ # Remove the container after it exists.
391
+ "--rm"
392
+
393
+ # Use the host network.
394
+ "--network=host"
395
+
396
+ # Run in priviledged mode. We are not using docker for sandboxing or
397
+ # isolation, just for packaging our dev tools.
398
+ "--privileged"
399
+
400
+ # Run the docker script with the user id. Because the docker image gets to
401
+ # write in ${PWD} you typically want this to be your user id.
402
+ # To allow docker in docker, we need to use docker gid on the host.
403
+ "--user" "${user_uid}:${docker_gid}"
404
+
405
+ # Pass down the USER.
406
+ "--env" "USER=${user_name}"
407
+
408
+ # Mount the project directory inside the Docker container.
409
+ "--volume" "${PROJECT_ROOT}:${TRAMPOLINE_WORKSPACE}"
410
+ "--workdir" "${TRAMPOLINE_WORKSPACE}"
411
+ "--env" "PROJECT_ROOT=${TRAMPOLINE_WORKSPACE}"
412
+
413
+ # Mount the temporary home directory.
414
+ "--volume" "${tmphome}:/h"
415
+ "--env" "HOME=/h"
416
+
417
+ # Allow docker in docker.
418
+ "--volume" "/var/run/docker.sock:/var/run/docker.sock"
419
+
420
+ # Mount the /tmp so that docker in docker can mount the files
421
+ # there correctly.
422
+ "--volume" "/tmp:/tmp"
423
+ # Pass down the KOKORO_GFILE_DIR and KOKORO_KEYSTORE_DIR
424
+ # TODO(tmatsuo): This part is not portable.
425
+ "--env" "TRAMPOLINE_SECRET_DIR=/secrets"
426
+ "--volume" "${KOKORO_GFILE_DIR:-/dev/shm}:/secrets/gfile"
427
+ "--env" "KOKORO_GFILE_DIR=/secrets/gfile"
428
+ "--volume" "${KOKORO_KEYSTORE_DIR:-/dev/shm}:/secrets/keystore"
429
+ "--env" "KOKORO_KEYSTORE_DIR=/secrets/keystore"
430
+ )
431
+
432
+ # Add an option for nicer output if the build gets a tty.
433
+ if [[ -t 0 ]]; then
434
+ docker_flags+=("-it")
435
+ fi
436
+
437
+ # Passing down env vars
438
+ for e in "${pass_down_envvars[@]}"
439
+ do
440
+ if [[ -n "${!e:-}" ]]; then
441
+ docker_flags+=("--env" "${e}=${!e}")
442
+ fi
443
+ done
444
+
445
+ # If arguments are given, all arguments will become the commands run
446
+ # in the container, otherwise run TRAMPOLINE_BUILD_FILE.
447
+ if [[ $# -ge 1 ]]; then
448
+ log_yellow "Running the given commands '" "${@:1}" "' in the container."
449
+ readonly commands=("${@:1}")
450
+ if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then
451
+ echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}"
452
+ fi
453
+ docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}" "${commands[@]}"
454
+ else
455
+ log_yellow "Running the tests in a Docker container."
456
+ docker_flags+=("--entrypoint=${TRAMPOLINE_BUILD_FILE}")
457
+ if [[ "${TRAMPOLINE_VERBOSE:-}" == "true" ]]; then
458
+ echo docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}"
459
+ fi
460
+ docker run "${docker_flags[@]}" "${TRAMPOLINE_IMAGE}"
461
+ fi
462
+
463
+
464
+ test_retval=$?
465
+
466
+ if [[ ${test_retval} -eq 0 ]]; then
467
+ log_green "Build finished with ${test_retval}"
468
+ else
469
+ log_red "Build finished with ${test_retval}"
470
+ fi
471
+
472
+ # Only upload it when the test passes.
473
+ if [[ "${update_cache}" == "true" ]] && \
474
+ [[ $test_retval == 0 ]] && \
475
+ [[ "${TRAMPOLINE_IMAGE_UPLOAD:-false}" == "true" ]]; then
476
+ log_yellow "Uploading the Docker image."
477
+ if docker push "${TRAMPOLINE_IMAGE}"; then
478
+ log_green "Finished uploading the Docker image."
479
+ else
480
+ log_red "Failed uploading the Docker image."
481
+ fi
482
+ # Call trampoline_after_upload_hook if it's defined.
483
+ if function_exists trampoline_after_upload_hook; then
484
+ trampoline_after_upload_hook
485
+ fi
486
+
487
+ fi
488
+
489
+ exit "${test_retval}"