seira 0.4.0 → 0.4.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/helpers.rb +6 -0
- data/lib/seira/app.rb +3 -7
- data/lib/seira/db.rb +205 -27
- data/lib/seira/jobs.rb +1 -1
- data/lib/seira/pods.rb +4 -1
- data/lib/seira/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 95d1b86a64920650d43cdc3d5063fadc2f17be89
|
4
|
+
data.tar.gz: d40cbb781664aeef22dfcadd4bc048ad27a69414
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 60c8eaaa70a5d8dad533e3bd1533f74bb7b59edbc7ce360ab68d43a4271d9eaf5c141e03ec2e7fbdb70ea917fcba5b8b32d5c45ab08108487fc84ce1ab2d34ef
|
7
|
+
data.tar.gz: aa114b74294c756197411515f4f94cd160187a00f9ef47c22f7d0fae181ff8089f2533266efe9aed62f989b5ff553c984300fa1e0af4d40ea3d0c2b68c577c94
|
data/lib/helpers.rb
CHANGED
data/lib/seira/app.rb
CHANGED
@@ -197,24 +197,20 @@ module Seira
|
|
197
197
|
puts "Copying source yaml from #{source} to temp folder"
|
198
198
|
FileUtils.mkdir_p destination # Create the nested directory
|
199
199
|
FileUtils.rm_rf("#{destination}/.", secure: true) # Clean out old files from the tmp folder
|
200
|
-
FileUtils.copy_entry source, destination
|
201
|
-
# Anything in jobs directory is not intended to be applied when deploying
|
202
|
-
# the app, but rather ran when needed as Job objects. Force to avoid exception if DNE.
|
203
|
-
FileUtils.rm_rf("#{destination}/jobs/") if File.directory?("#{destination}/jobs/")
|
204
200
|
|
205
201
|
# Iterate through each yaml file and find/replace and save
|
206
202
|
puts "Iterating temp folder files find/replace revision information"
|
207
|
-
Dir.foreach(
|
203
|
+
Dir.foreach(source) do |item|
|
208
204
|
next if item == '.' || item == '..'
|
209
205
|
|
210
206
|
# If we have run into a directory item, skip it
|
211
|
-
next if File.directory?("#{
|
207
|
+
next if File.directory?("#{source}/#{item}")
|
212
208
|
|
213
209
|
# Skip any manifest file that has "seira-skip.yaml" at the end. Common use case is for Job definitions
|
214
210
|
# to be used in "seira staging <app> jobs run"
|
215
211
|
next if item.end_with?("seira-skip.yaml")
|
216
212
|
|
217
|
-
text = File.read("#{
|
213
|
+
text = File.read("#{source}/#{item}")
|
218
214
|
|
219
215
|
new_contents = text
|
220
216
|
replacement_hash.each do |key, value|
|
data/lib/seira/db.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'securerandom'
|
2
|
+
require 'English'
|
2
3
|
|
3
4
|
require_relative 'db/create'
|
4
5
|
|
@@ -6,7 +7,7 @@ module Seira
|
|
6
7
|
class Db
|
7
8
|
include Seira::Commands
|
8
9
|
|
9
|
-
VALID_ACTIONS = %w[help create delete list restart connect ps kill analyze create-readonly-user].freeze
|
10
|
+
VALID_ACTIONS = %w[help create delete list restart connect ps kill analyze create-readonly-user psql table-sizes index-sizes vacuum unused-indexes unused-indices user-connections info].freeze
|
10
11
|
SUMMARY = "Manage your Cloud SQL Postgres databases.".freeze
|
11
12
|
|
12
13
|
attr_reader :app, :action, :args, :context
|
@@ -40,6 +41,20 @@ module Seira
|
|
40
41
|
run_analyze
|
41
42
|
when 'create-readonly-user'
|
42
43
|
run_create_readonly_user
|
44
|
+
when 'psql'
|
45
|
+
run_psql
|
46
|
+
when 'table-sizes'
|
47
|
+
run_table_sizes
|
48
|
+
when 'index-sizes'
|
49
|
+
run_index_sizes
|
50
|
+
when 'vacuum'
|
51
|
+
run_vacuum
|
52
|
+
when 'unused-indexes', 'unused-indices'
|
53
|
+
run_unused_indexes
|
54
|
+
when 'user-connections'
|
55
|
+
run_user_connections
|
56
|
+
when 'info'
|
57
|
+
run_info
|
43
58
|
else
|
44
59
|
fail "Unknown command encountered"
|
45
60
|
end
|
@@ -62,15 +77,24 @@ module Seira
|
|
62
77
|
def run_help
|
63
78
|
puts SUMMARY
|
64
79
|
puts "\n"
|
65
|
-
puts
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
80
|
+
puts <<~HELPTEXT
|
81
|
+
analyze: Display database performance information
|
82
|
+
connect: Open a psql command prompt via gcloud connect. You will be shown the password needed before the prompt opens.
|
83
|
+
create: Create a new postgres instance in cloud sql. Supports creating replicas and other numerous flags.
|
84
|
+
create-readonly-user: Create a database user named by --username=<name> with only SELECT access privileges
|
85
|
+
delete: Delete a postgres instance from cloud sql. Use with caution, and remove all kubernetes configs first.
|
86
|
+
index-sizes: List sizes of all indexes in the database
|
87
|
+
info: Summarize all database instances for the app
|
88
|
+
kill: Kill a query
|
89
|
+
list: List all postgres instances.
|
90
|
+
ps: List running queries
|
91
|
+
psql: Open a psql prompt via kubectl exec into a pgbouncer pod.
|
92
|
+
restart: Fully restart the database.
|
93
|
+
table-sizes: List sizes of all tables in the database
|
94
|
+
unused-indexes: Show indexes with zero or low usage
|
95
|
+
user-connections: List number of connections per user
|
96
|
+
vacuum: Run a VACUUM ANALYZE
|
97
|
+
HELPTEXT
|
74
98
|
end
|
75
99
|
|
76
100
|
def run_create
|
@@ -120,21 +144,22 @@ module Seira
|
|
120
144
|
end
|
121
145
|
end
|
122
146
|
|
123
|
-
execute_db_command(
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
147
|
+
execute_db_command(
|
148
|
+
<<~SQL
|
149
|
+
SELECT
|
150
|
+
pid,
|
151
|
+
state,
|
152
|
+
application_name AS source,
|
153
|
+
age(now(),query_start) AS running_for,
|
154
|
+
query_start,
|
155
|
+
wait_event IS NOT NULL AS waiting,
|
156
|
+
query
|
157
|
+
FROM pg_stat_activity
|
158
|
+
WHERE
|
159
|
+
query <> '<insufficient privilege>'
|
160
|
+
#{verbose ? '' : "AND state <> 'idle'"}
|
161
|
+
AND pid <> pg_backend_pid()
|
162
|
+
ORDER BY query_start DESC
|
138
163
|
SQL
|
139
164
|
)
|
140
165
|
end
|
@@ -235,7 +260,148 @@ module Seira
|
|
235
260
|
execute_db_command(database_commands)
|
236
261
|
end
|
237
262
|
|
238
|
-
def
|
263
|
+
def run_psql
|
264
|
+
execute_db_command(nil, interactive: true)
|
265
|
+
end
|
266
|
+
|
267
|
+
def run_table_sizes
|
268
|
+
# https://wiki.postgresql.org/wiki/Disk_Usage
|
269
|
+
execute_db_command(
|
270
|
+
<<~SQL
|
271
|
+
SELECT table_name
|
272
|
+
, row_estimate
|
273
|
+
, pg_size_pretty(table_bytes) AS table
|
274
|
+
, pg_size_pretty(index_bytes) AS index
|
275
|
+
, pg_size_pretty(toast_bytes) AS toast
|
276
|
+
, pg_size_pretty(total_bytes) AS total
|
277
|
+
FROM (
|
278
|
+
SELECT *, total_bytes-index_bytes-COALESCE(toast_bytes,0) AS table_bytes FROM (
|
279
|
+
SELECT relname AS table_name
|
280
|
+
, c.reltuples AS row_estimate
|
281
|
+
, pg_total_relation_size(c.oid) AS total_bytes
|
282
|
+
, pg_indexes_size(c.oid) AS index_bytes
|
283
|
+
, pg_total_relation_size(reltoastrelid) AS toast_bytes
|
284
|
+
FROM pg_class c
|
285
|
+
LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
|
286
|
+
WHERE relkind = 'r'
|
287
|
+
AND n.nspname = 'public'
|
288
|
+
) a
|
289
|
+
) a
|
290
|
+
ORDER BY total_bytes DESC;
|
291
|
+
SQL
|
292
|
+
)
|
293
|
+
end
|
294
|
+
|
295
|
+
def run_index_sizes
|
296
|
+
# https://wiki.postgresql.org/wiki/Disk_Usage
|
297
|
+
execute_db_command(
|
298
|
+
<<~SQL
|
299
|
+
SELECT relname AS index
|
300
|
+
, c.reltuples AS row_estimate
|
301
|
+
, pg_size_pretty(pg_relation_size(c.oid)) AS "size"
|
302
|
+
FROM pg_class c
|
303
|
+
LEFT JOIN pg_namespace n ON (n.oid = c.relnamespace)
|
304
|
+
WHERE relkind = 'i'
|
305
|
+
AND n.nspname = 'public'
|
306
|
+
ORDER BY pg_relation_size(c.oid) DESC;
|
307
|
+
SQL
|
308
|
+
)
|
309
|
+
end
|
310
|
+
|
311
|
+
def run_vacuum
|
312
|
+
execute_db_command(
|
313
|
+
<<~SQL
|
314
|
+
VACUUM VERBOSE ANALYZE;
|
315
|
+
SQL
|
316
|
+
)
|
317
|
+
end
|
318
|
+
|
319
|
+
def run_unused_indexes
|
320
|
+
# https://github.com/heroku/heroku-pg-extras/blob/master/commands/unused_indexes.js
|
321
|
+
execute_db_command(
|
322
|
+
<<~SQL
|
323
|
+
SELECT
|
324
|
+
schemaname || '.' || relname AS table,
|
325
|
+
indexrelname AS index,
|
326
|
+
pg_size_pretty(pg_relation_size(i.indexrelid)) AS index_size,
|
327
|
+
idx_scan as index_scans
|
328
|
+
FROM pg_stat_user_indexes ui
|
329
|
+
JOIN pg_index i ON ui.indexrelid = i.indexrelid
|
330
|
+
WHERE NOT indisunique AND idx_scan < 50 AND pg_relation_size(relid) > 5 * 8192
|
331
|
+
ORDER BY pg_relation_size(i.indexrelid) / nullif(idx_scan, 0) DESC NULLS FIRST,
|
332
|
+
pg_relation_size(i.indexrelid) DESC;
|
333
|
+
SQL
|
334
|
+
)
|
335
|
+
end
|
336
|
+
|
337
|
+
def run_user_connections
|
338
|
+
execute_db_command(
|
339
|
+
<<~SQL
|
340
|
+
SELECT usename AS user, count(pid) FROM pg_stat_activity GROUP BY usename;
|
341
|
+
SQL
|
342
|
+
)
|
343
|
+
end
|
344
|
+
|
345
|
+
def run_info
|
346
|
+
instances = JSON.parse(gcloud("sql instances list --filter='name~\\A#{app}-'", context: context, format: :json))
|
347
|
+
instances.each do |instance|
|
348
|
+
db_info_command =
|
349
|
+
<<~SQL
|
350
|
+
COPY (SELECT pg_size_pretty(sum(pg_database_size(datname))) FROM pg_database) TO stdout;
|
351
|
+
COPY (SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE') TO stdout;
|
352
|
+
COPY (SELECT count(*) FROM pg_stat_activity) TO stdout;
|
353
|
+
SQL
|
354
|
+
db_info = execute_db_command(db_info_command, print: false)
|
355
|
+
data_size, table_count, connection_count = db_info.split("\n")
|
356
|
+
instance['data_size'] = data_size
|
357
|
+
instance['table_count'] = table_count
|
358
|
+
instance['connection_count'] = connection_count
|
359
|
+
end
|
360
|
+
instances.each do |instance|
|
361
|
+
# https://cloud.google.com/sql/faq
|
362
|
+
disk_size = instance['settings']['dataDiskSizeGb'].to_f
|
363
|
+
connection_limit =
|
364
|
+
if disk_size <= 0.6
|
365
|
+
25
|
366
|
+
elsif disk_size <= 3.75
|
367
|
+
50
|
368
|
+
elsif disk_size <= 6
|
369
|
+
100
|
370
|
+
elsif disk_size <= 7.5
|
371
|
+
150
|
372
|
+
elsif disk_size <= 15
|
373
|
+
200
|
374
|
+
elsif disk_size <= 30
|
375
|
+
250
|
376
|
+
elsif disk_size <= 60
|
377
|
+
300
|
378
|
+
elsif disk_size <= 120
|
379
|
+
400
|
380
|
+
else
|
381
|
+
500
|
382
|
+
end
|
383
|
+
|
384
|
+
backup_info = instance['settings']['backupConfiguration']['enabled'] == 'true' ? instance['settings']['backupConfiguration']['startTime'] : 'false'
|
385
|
+
|
386
|
+
puts "\n"
|
387
|
+
puts instance['name'].bold
|
388
|
+
puts <<~INFOTEXT
|
389
|
+
State: #{instance['state']}
|
390
|
+
Tables: #{instance['table_count']}
|
391
|
+
Disk Size: #{disk_size} GB
|
392
|
+
Data Size: #{instance['data_size']}
|
393
|
+
Auto Resize: #{instance['settings']['storageAutoResize']}
|
394
|
+
Disk Type: #{instance['settings']['dataDiskType']}
|
395
|
+
Tier: #{instance['settings']['tier']}
|
396
|
+
Availability: #{instance['settings']['availabilityType']}
|
397
|
+
Version: #{instance['databaseVersion']}
|
398
|
+
Backups: #{backup_info}
|
399
|
+
Connections: #{instance['connection_count']}/#{connection_limit}(?)
|
400
|
+
INFOTEXT
|
401
|
+
end
|
402
|
+
end
|
403
|
+
|
404
|
+
def execute_db_command(sql_command, as_admin: false, interactive: false, print: true)
|
239
405
|
# TODO(josh): move pgbouncer naming logic here and in Create to a common location
|
240
406
|
instance_name = primary_instance
|
241
407
|
tier = instance_name.gsub("#{app}-", '')
|
@@ -252,7 +418,19 @@ module Seira
|
|
252
418
|
else
|
253
419
|
'psql'
|
254
420
|
end
|
255
|
-
|
421
|
+
system_command = "kubectl exec #{pod_name} --namespace #{app}"
|
422
|
+
system_command += ' -ti' if interactive
|
423
|
+
system_command += " -- #{psql_command}"
|
424
|
+
system_command += " -c \"#{sql_command}\"" unless sql_command.nil?
|
425
|
+
if interactive
|
426
|
+
exit(1) unless system(system_command)
|
427
|
+
else
|
428
|
+
output = `#{system_command}`
|
429
|
+
success = $CHILD_STATUS.success?
|
430
|
+
puts output if print || !success
|
431
|
+
exit(1) unless success
|
432
|
+
output
|
433
|
+
end
|
256
434
|
end
|
257
435
|
|
258
436
|
def existing_instances(remove_app_prefix: true)
|
data/lib/seira/jobs.rb
CHANGED
@@ -84,7 +84,7 @@ module Seira
|
|
84
84
|
replacement_hash = {
|
85
85
|
'UNIQUE_NAME' => unique_name,
|
86
86
|
'REVISION' => revision,
|
87
|
-
'COMMAND' =>
|
87
|
+
'COMMAND' => %("sh", "-c", "#{command}"),
|
88
88
|
'CPU_REQUEST' => '200m',
|
89
89
|
'CPU_LIMIT' => '500m',
|
90
90
|
'MEMORY_REQUEST' => '500Mi',
|
data/lib/seira/pods.rb
CHANGED
data/lib/seira/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: seira
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Scott Ringwelski
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-04-
|
11
|
+
date: 2018-04-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: highline
|