pgdexter 0.3.4 → 0.3.9

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e51c5c9ce5c31760d773177a3b224674752b19de8cc9fa0978e1da2adff9a490
4
- data.tar.gz: ce9b2e89d2524aafda39bd5a91bb53abde5d0819e0c8751281a5ff7ad10766c2
3
+ metadata.gz: 724c998e7bee78818829b6a7f45134869f5f336f956467c59f7e8396724b37a6
4
+ data.tar.gz: dd68dfde6648309a5cf054829e74f762724da91ef4d79200988dfe3eff4a3599
5
5
  SHA512:
6
- metadata.gz: 63cdf589a11ab232f45e26ff8b6a713ef757ce3bc3baf3fecbc51217fb4c3a4bdda0ded52bc631bbb8aaa5fe7ca28d356ab99e200c5af22a3bd1bac9e5efc17b
7
- data.tar.gz: fcbb680a98f5907a62216ee142fa91b8538965c1fd59c1de48645ee1b5e3525950245d010a128f52b28fda665d0f6fcbf95fa343184b559150646819049f173a
6
+ metadata.gz: c658bdf247b47c6419f5add3fdbd5becf8a5eaa915a7cd76e6ebd09e441fbbf099963ea78d6fe26788e8fdaa49b14e883e283f5b713930a942cdd55cd600cc1c
7
+ data.tar.gz: 55384f3374359b4e713ed903c3cb1cebd4e06197d903ecac802383a2358fc712646cad10bc59abf7fc6558d9175d91137bedc5861dfe3b1b6f873c5bf9872504
@@ -1,28 +1,51 @@
1
- ## 0.3.4
1
+ ## 0.3.9 (2020-11-23)
2
+
3
+ - Added `--tablespace` option
4
+
5
+ ## 0.3.8 (2020-08-17)
6
+
7
+ - Colorize output
8
+ - Fixed error when unable to parse view definitions
9
+
10
+ ## 0.3.7 (2020-07-10)
11
+
12
+ - Fixed help output
13
+
14
+ ## 0.3.6 (2020-03-30)
15
+
16
+ - Fixed warning with Ruby 2.7
17
+
18
+ ## 0.3.5 (2018-04-30)
19
+
20
+ - Added `sql` input format
21
+ - Fixed error for queries with double dash comments
22
+ - Fixed connection threading issue with `--pg-stat-activity` option
23
+
24
+ ## 0.3.4 (2018-04-09)
2
25
 
3
26
  - Fixed `--username` option
4
27
  - Fixed `JSON::NestingError`
5
28
  - Added `--pg-stat-activity` option
6
29
 
7
- ## 0.3.3
30
+ ## 0.3.3 (2018-02-22)
8
31
 
9
32
  - Added support for views and materialized views
10
33
  - Better handle case when multiple indexes are found for a query
11
34
  - Added `--min-cost-savings-pct` option
12
35
 
13
- ## 0.3.2
36
+ ## 0.3.2 (2018-01-04)
14
37
 
15
38
  - Fixed parsing issue with named prepared statements
16
39
  - Fixed parsing issue with multiline queries in csv format
17
40
  - Better explanations for indexing decisions
18
41
 
19
- ## 0.3.1
42
+ ## 0.3.1 (2017-12-28)
20
43
 
21
44
  - Added support for queries with bind variables
22
45
  - Fixed error with streaming logs as csv format
23
46
  - Handle malformed CSV gracefully
24
47
 
25
- ## 0.3.0
48
+ ## 0.3.0 (2017-12-22)
26
49
 
27
50
  - Added support for schemas
28
51
  - Added support for csv format
@@ -30,12 +53,12 @@
30
53
  - Added `--min-calls` option
31
54
  - Fixed debug output when indexes not found
32
55
 
33
- ## 0.2.1
56
+ ## 0.2.1 (2017-09-02)
34
57
 
35
58
  - Fixed bad suggestions
36
59
  - Improved debugging output
37
60
 
38
- ## 0.2.0
61
+ ## 0.2.0 (2017-08-27)
39
62
 
40
63
  - Added same connection options as `psql`
41
64
  - Added support for multiple files
@@ -46,38 +69,38 @@ Breaking
46
69
 
47
70
  - `-h` option changed to `--host` instead of `--help` for consistency with `psql`
48
71
 
49
- ## 0.1.6
72
+ ## 0.1.6 (2017-08-26)
50
73
 
51
74
  - Significant performance improvements
52
75
  - Added `--include` option
53
76
 
54
- ## 0.1.5
77
+ ## 0.1.5 (2017-08-14)
55
78
 
56
79
  - Added support for non-`SELECT` queries
57
80
  - Added `--pg-stat-statements` option
58
81
  - Added advisory locks
59
82
  - Added support for running as a non-superuser
60
83
 
61
- ## 0.1.4
84
+ ## 0.1.4 (2017-07-02)
62
85
 
63
86
  - Added support for multicolumn indexes
64
87
 
65
- ## 0.1.3
88
+ ## 0.1.3 (2017-06-30)
66
89
 
67
90
  - Fixed error with non-lowercase columns
68
91
  - Fixed error with `json` columns
69
92
 
70
- ## 0.1.2
93
+ ## 0.1.2 (2017-06-26)
71
94
 
72
95
  - Added `--exclude` option
73
96
  - Added `--log-sql` option
74
97
 
75
- ## 0.1.1
98
+ ## 0.1.1 (2017-06-25)
76
99
 
77
100
  - Added `--interval` option
78
101
  - Added `--min-time` option
79
102
  - Added `--log-level` option
80
103
 
81
- ## 0.1.0
104
+ ## 0.1.0 (2017-06-24)
82
105
 
83
106
  - Launched
@@ -1,4 +1,4 @@
1
- Copyright (c) 2017 Andrew Kane
1
+ Copyright (c) 2017-2020 Andrew Kane
2
2
 
3
3
  MIT License
4
4
 
data/README.md CHANGED
@@ -2,9 +2,9 @@
2
2
 
3
3
  The automatic indexer for Postgres
4
4
 
5
- [Read about how it works](https://medium.com/@ankane/introducing-dexter-the-automatic-indexer-for-postgres-5f8fa8b28f27)
5
+ [Read about how it works](https://ankane.org/introducing-dexter)
6
6
 
7
- [![Build Status](https://travis-ci.org/ankane/dexter.svg?branch=master)](https://travis-ci.org/ankane/dexter)
7
+ [![Build Status](https://github.com/ankane/dexter/workflows/build/badge.svg?branch=master)](https://github.com/ankane/dexter/actions)
8
8
 
9
9
  ## Installation
10
10
 
@@ -12,8 +12,8 @@ First, install [HypoPG](https://github.com/HypoPG/hypopg) on your database serve
12
12
 
13
13
  ```sh
14
14
  cd /tmp
15
- curl -L https://github.com/HypoPG/hypopg/archive/1.1.1.tar.gz | tar xz
16
- cd hypopg-1.1.1
15
+ curl -L https://github.com/HypoPG/hypopg/archive/1.1.4.tar.gz | tar xz
16
+ cd hypopg-1.1.4
17
17
  make
18
18
  make install # may need sudo
19
19
  ```
@@ -104,7 +104,7 @@ or pass files:
104
104
  dexter <connection-options> <file1> <file2>
105
105
  ```
106
106
 
107
- or collect running queries with: [master]
107
+ or collect running queries with:
108
108
 
109
109
  ```sh
110
110
  dexter <connection-options> --pg-stat-activity
@@ -146,10 +146,10 @@ dexter --interval 60 # seconds
146
146
 
147
147
  ## Examples
148
148
 
149
- Ubuntu with PostgreSQL 9.6
149
+ Ubuntu with PostgreSQL 12
150
150
 
151
151
  ```sh
152
- tail -F -n +1 /var/log/postgresql/postgresql-9.6-main.log | sudo -u postgres dexter dbname
152
+ tail -F -n +1 /var/log/postgresql/postgresql-12-main.log | sudo -u postgres dexter dbname
153
153
  ```
154
154
 
155
155
  Homebrew on Mac
@@ -215,6 +215,10 @@ gem specific_install https://github.com/ankane/dexter.git
215
215
 
216
216
  This software wouldn’t be possible without [HypoPG](https://github.com/dalibo/hypopg), which allows you to create hypothetical indexes, and [pg_query](https://github.com/lfittl/pg_query), which allows you to parse and fingerprint queries. A big thanks to Dalibo and Lukas Fittl respectively.
217
217
 
218
+ ## Research
219
+
220
+ This is known as the Index Selection Problem (ISP).
221
+
218
222
  ## Contributing
219
223
 
220
224
  Everyone is encouraged to help improve this project. Here are a few ways you can help:
@@ -224,17 +228,18 @@ Everyone is encouraged to help improve this project. Here are a few ways you can
224
228
  - Write, clarify, or fix documentation
225
229
  - Suggest or add new features
226
230
 
227
- To get started, run:
231
+ To get started with development, run:
228
232
 
229
233
  ```sh
230
234
  git clone https://github.com/ankane/dexter.git
231
235
  cd dexter
232
- bundle
233
- rake install
236
+ bundle install
237
+ bundle exec rake install
234
238
  ```
235
239
 
236
240
  To run tests, use:
237
241
 
238
242
  ```sh
239
- rake test
243
+ createdb dexter_test
244
+ bundle exec rake test
240
245
  ```
data/exe/dexter CHANGED
@@ -1,10 +1,7 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
+ # handle interrupts
4
+ trap("SIGINT") { abort }
5
+
3
6
  require "dexter"
4
- begin
5
- Dexter::Client.new(ARGV).perform
6
- rescue Dexter::Abort => e
7
- abort e.message
8
- rescue Interrupt => e
9
- # do nothing
10
- end
7
+ Dexter::Client.start
@@ -1,10 +1,14 @@
1
- require "dexter/version"
2
- require "slop"
1
+ # dependencies
3
2
  require "pg"
4
3
  require "pg_query"
5
- require "time"
4
+ require "slop"
5
+
6
+ # stdlib
6
7
  require "set"
7
- require "thread"
8
+ require "time"
9
+
10
+ # modules
11
+ require "dexter/version"
8
12
  require "dexter/logging"
9
13
  require "dexter/client"
10
14
  require "dexter/collector"
@@ -12,6 +16,7 @@ require "dexter/indexer"
12
16
  require "dexter/log_parser"
13
17
  require "dexter/csv_log_parser"
14
18
  require "dexter/pg_stat_activity_parser"
19
+ require "dexter/sql_log_parser"
15
20
  require "dexter/processor"
16
21
  require "dexter/query"
17
22
 
@@ -1,9 +1,16 @@
1
1
  module Dexter
2
2
  class Client
3
+ extend Logging
3
4
  include Logging
4
5
 
5
6
  attr_reader :arguments, :options
6
7
 
8
+ def self.start
9
+ Dexter::Client.new(ARGV).perform
10
+ rescue Dexter::Abort, PG::UndefinedFile => e
11
+ abort colorize(e.message.strip, :red)
12
+ end
13
+
7
14
  def initialize(args)
8
15
  @arguments, @options = parse_args(args)
9
16
  end
@@ -31,9 +38,9 @@ module Dexter
31
38
  def parse_args(args)
32
39
  opts = Slop.parse(args) do |o|
33
40
  o.banner = %(Usage:
34
- dexter [options]
35
-
36
- Options:)
41
+ dexter [options])
42
+ o.separator ""
43
+ o.separator "Options:"
37
44
  o.boolean "--analyze", "analyze tables that haven't been analyzed in the past hour", default: false
38
45
  o.boolean "--create", "create indexes", default: false
39
46
  o.array "--exclude", "prevent specific tables from being indexed"
@@ -49,9 +56,7 @@ Options:)
49
56
  o.boolean "--pg-stat-activity", "use pg_stat_activity", default: false, help: false
50
57
  o.boolean "--pg-stat-statements", "use pg_stat_statements", default: false, help: false
51
58
  o.string "-s", "--statement", "process a single statement"
52
- # separator must go here to show up correctly - slop bug?
53
- o.separator ""
54
- o.separator "Connection options:"
59
+ o.string "--tablespace", "tablespace to create indexes"
55
60
  o.on "-v", "--version", "print the version" do
56
61
  log Dexter::VERSION
57
62
  exit
@@ -60,10 +65,12 @@ Options:)
60
65
  log o
61
66
  exit
62
67
  end
63
- o.string "-U", "--username"
64
- o.string "-d", "--dbname"
65
- o.string "-h", "--host"
66
- o.integer "-p", "--port"
68
+ o.separator ""
69
+ o.separator "Connection options:"
70
+ o.string "-d", "--dbname", "database name"
71
+ o.string "-h", "--host", "database host"
72
+ o.integer "-p", "--port", "database port"
73
+ o.string "-U", "--username", "database user"
67
74
  end
68
75
 
69
76
  arguments = opts.arguments
@@ -73,11 +80,11 @@ Options:)
73
80
 
74
81
  # TODO don't use global var
75
82
  $log_level = options[:log_level].to_s.downcase
76
- abort "Unknown log level" unless ["error", "info", "debug", "debug2", "debug3"].include?($log_level)
83
+ raise Dexter::Abort, "Unknown log level" unless ["error", "info", "debug", "debug2", "debug3"].include?($log_level)
77
84
 
78
85
  [arguments, options]
79
86
  rescue Slop::Error => e
80
- abort e.message
87
+ raise Dexter::Abort, e.message
81
88
  end
82
89
  end
83
90
  end
@@ -16,7 +16,7 @@ module Dexter
16
16
  end
17
17
  end
18
18
  rescue CSV::MalformedCSVError => e
19
- abort "ERROR: #{e.message}"
19
+ raise Dexter::Abort, "ERROR: #{e.message}"
20
20
  end
21
21
  end
22
22
  end
@@ -4,6 +4,7 @@ module Dexter
4
4
 
5
5
  def initialize(options)
6
6
  @create = options[:create]
7
+ @tablespace = options[:tablespace]
7
8
  @log_level = options[:log_level]
8
9
  @exclude_tables = options[:exclude]
9
10
  @include_tables = Array(options[:include].split(",")) if options[:include]
@@ -14,6 +15,7 @@ module Dexter
14
15
  @analyze = options[:analyze]
15
16
  @min_cost_savings_pct = options[:min_cost_savings_pct].to_i
16
17
  @options = options
18
+ @mutex = Mutex.new
17
19
 
18
20
  create_extension unless extension_exists?
19
21
  execute("SET lock_timeout = '5s'")
@@ -106,13 +108,13 @@ module Dexter
106
108
  analyze_tables(tables) if tables.any? && (@analyze || @log_level == "debug2")
107
109
 
108
110
  # create hypothetical indexes and explain queries
109
- candidates = tables.any? ? create_hypothetical_indexes(queries.select(&:candidate_tables), tables) : {}
111
+ candidates = tables.any? ? create_hypothetical_indexes(queries.select(&:candidate_tables)) : {}
110
112
 
111
113
  # see if new indexes were used and meet bar
112
114
  new_indexes = determine_indexes(queries, candidates, tables)
113
115
 
114
116
  # display and create new indexes
115
- show_and_create_indexes(new_indexes, queries, tables)
117
+ show_and_create_indexes(new_indexes, queries)
116
118
  end
117
119
 
118
120
  private
@@ -122,9 +124,9 @@ module Dexter
122
124
  begin
123
125
  execute("CREATE EXTENSION IF NOT EXISTS hypopg")
124
126
  rescue PG::UndefinedFile
125
- abort "Install HypoPG first: https://github.com/ankane/dexter#installation"
127
+ raise Dexter::Abort, "Install HypoPG first: https://github.com/ankane/dexter#installation"
126
128
  rescue PG::InsufficientPrivilege
127
- abort "Use a superuser to run: CREATE EXTENSION hypopg"
129
+ raise Dexter::Abort, "Use a superuser to run: CREATE EXTENSION hypopg"
128
130
  end
129
131
  end
130
132
 
@@ -181,7 +183,7 @@ module Dexter
181
183
  query.plans << plan(query.statement)
182
184
  if @log_explain
183
185
  # Pass format to prevent ANALYZE
184
- puts execute("EXPLAIN (FORMAT TEXT) #{safe_statement(query.statement)}").map { |r| r["QUERY PLAN"] }.join("\n")
186
+ puts execute("EXPLAIN (FORMAT TEXT) #{safe_statement(query.statement)}", pretty: false).map { |r| r["QUERY PLAN"] }.join("\n")
185
187
  end
186
188
  rescue PG::Error, JSON::NestingError => e
187
189
  if @log_explain
@@ -192,7 +194,7 @@ module Dexter
192
194
  end
193
195
  end
194
196
 
195
- def create_hypothetical_indexes(queries, tables)
197
+ def create_hypothetical_indexes(queries)
196
198
  candidates = {}
197
199
 
198
200
  # get initial costs for queries
@@ -362,11 +364,12 @@ module Dexter
362
364
  winning_cost < query.initial_cost * savings_ratio
363
365
  end
364
366
 
367
+ query_indexes = [winning_index]
368
+ new_cost3 = winning_cost
369
+ query.pass3_indexes = query_indexes
370
+
365
371
  if use_winning
366
- query_indexes = [winning_index]
367
372
  cost_savings3 = true
368
- new_cost3 = winning_cost
369
- query.pass3_indexes = query_indexes
370
373
  else
371
374
  suggest_index = false
372
375
  end
@@ -415,11 +418,11 @@ module Dexter
415
418
  end
416
419
  end
417
420
 
418
- def show_and_create_indexes(new_indexes, queries, tables)
421
+ def show_and_create_indexes(new_indexes, queries)
419
422
  # print summary
420
423
  if new_indexes.any?
421
424
  new_indexes.each do |index|
422
- log "Index found: #{index[:table]} (#{index[:columns].join(", ")})"
425
+ log colorize("Index found: #{index[:table]} (#{index[:columns].join(", ")})", :green)
423
426
  end
424
427
  else
425
428
  log "No new indexes found"
@@ -456,7 +459,7 @@ module Dexter
456
459
  log "Pass3: #{query.costs[3]} : #{log_indexes(query.pass3_indexes || [])}"
457
460
  end
458
461
  log "Final: #{query.new_cost} : #{log_indexes(query.suggest_index ? query_indexes : [])}"
459
- if query_indexes.size == 1 && !query.suggest_index
462
+ if (query.pass1_indexes.any? || query.pass2_indexes.any?) && !query.suggest_index
460
463
  log "Need #{@min_cost_savings_pct}% cost savings to suggest index"
461
464
  end
462
465
  else
@@ -477,7 +480,8 @@ module Dexter
477
480
  with_advisory_lock do
478
481
  new_indexes.each do |index|
479
482
  unless index_exists?(index)
480
- statement = "CREATE INDEX CONCURRENTLY ON #{quote_ident(index[:table])} (#{index[:columns].map { |c| quote_ident(c) }.join(", ")})"
483
+ statement = String.new("CREATE INDEX CONCURRENTLY ON #{quote_ident(index[:table])} (#{index[:columns].map { |c| quote_ident(c) }.join(", ")})")
484
+ statement << " TABLESPACE #{quote_ident(@tablespace)}" if @tablespace
481
485
  log "Creating index: #{statement}"
482
486
  started_at = Time.now
483
487
  begin
@@ -513,10 +517,10 @@ module Dexter
513
517
  PG::Connection.new(config)
514
518
  end
515
519
  rescue PG::ConnectionBad => e
516
- abort e.message
520
+ raise Dexter::Abort, e.message
517
521
  end
518
522
 
519
- def execute(query)
523
+ def execute(query, pretty: true)
520
524
  # use exec_params instead of exec for security
521
525
  #
522
526
  # Unlike PQexec, PQexecParams allows at most one SQL command in the given string.
@@ -524,14 +528,17 @@ module Dexter
524
528
  # This is a limitation of the underlying protocol, but has some usefulness
525
529
  # as an extra defense against SQL-injection attacks.
526
530
  # https://www.postgresql.org/docs/current/static/libpq-exec.html
527
- query = squish(query)
528
- log "SQL: #{query}" if @log_sql
529
- conn.exec_params(query, []).to_a
531
+ query = squish(query) if pretty
532
+ log colorize("[sql] #{query}", :cyan) if @log_sql
533
+
534
+ @mutex.synchronize do
535
+ conn.exec_params(query, []).to_a
536
+ end
530
537
  end
531
538
 
532
539
  def plan(query)
533
540
  # strip semi-colons as another measure of defense
534
- JSON.parse(execute("EXPLAIN (FORMAT JSON) #{safe_statement(query)}").first["QUERY PLAN"], max_nesting: 1000).first["Plan"]
541
+ JSON.parse(execute("EXPLAIN (FORMAT JSON) #{safe_statement(query)}", pretty: false).first["QUERY PLAN"], max_nesting: 1000).first["Plan"]
535
542
  end
536
543
 
537
544
  # TODO for multicolumn indexes, use ordering
@@ -591,7 +598,13 @@ module Dexter
591
598
 
592
599
  view_tables = {}
593
600
  result.each do |row|
594
- view_tables[row["table_name"]] = PgQuery.parse(row["definition"]).tables
601
+ begin
602
+ view_tables[row["table_name"]] = PgQuery.parse(row["definition"]).tables
603
+ rescue PgQuery::ParseError
604
+ if @log_level.start_with?("debug")
605
+ log colorize("ERROR: Cannot parse view definition: #{row["table_name"]}", :red)
606
+ end
607
+ end
595
608
  end
596
609
 
597
610
  view_tables
@@ -1,11 +1,26 @@
1
1
  module Dexter
2
2
  module Logging
3
+ COLOR_CODES = {
4
+ red: 31,
5
+ green: 32,
6
+ yellow: 33,
7
+ cyan: 36
8
+ }
9
+
10
+ def output
11
+ $stdout
12
+ end
13
+
3
14
  def log(message = "")
4
- puts message unless $log_level == "error"
15
+ output.puts(message) unless $log_level == "error"
5
16
  end
6
17
 
7
- def abort(message)
8
- raise Dexter::Abort, message
18
+ def colorize(message, color)
19
+ if output.tty?
20
+ "\e[#{COLOR_CODES[color]}m#{message}\e[0m"
21
+ else
22
+ message
23
+ end
9
24
  end
10
25
  end
11
26
  end
@@ -18,7 +18,7 @@ module Dexter
18
18
 
19
19
  queries = new_queries
20
20
 
21
- sleep(5)
21
+ sleep(1)
22
22
  end
23
23
  end
24
24
  end
@@ -13,6 +13,8 @@ module Dexter
13
13
  PgStatActivityParser.new(@indexer, @collector)
14
14
  elsif options[:input_format] == "csv"
15
15
  CsvLogParser.new(logfile, @collector)
16
+ elsif options[:input_format] == "sql"
17
+ SqlLogParser.new(logfile, @collector)
16
18
  else
17
19
  LogParser.new(logfile, @collector)
18
20
  end
@@ -35,7 +37,7 @@ module Dexter
35
37
  begin
36
38
  process_queries
37
39
  rescue PG::ServerError => e
38
- log "ERROR: #{e.class.name}: #{e.message}"
40
+ log colorize("ERROR: #{e.class.name}: #{e.message}", :red)
39
41
  end
40
42
  sleep(@interval)
41
43
  end
@@ -45,7 +47,7 @@ module Dexter
45
47
  begin
46
48
  @log_parser.perform
47
49
  rescue Errno::ENOENT => e
48
- abort "ERROR: #{e.message}"
50
+ raise Dexter::Abort, "ERROR: #{e.message}"
49
51
  end
50
52
 
51
53
  process_queries
@@ -0,0 +1,10 @@
1
+ module Dexter
2
+ class SqlLogParser < LogParser
3
+ def perform
4
+ # TODO support streaming
5
+ @logfile.read.split(";").each do |statement|
6
+ process_entry(statement, 0)
7
+ end
8
+ end
9
+ end
10
+ end
@@ -1,3 +1,3 @@
1
1
  module Dexter
2
- VERSION = "0.3.4"
2
+ VERSION = "0.3.9"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pgdexter
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.4
4
+ version: 0.3.9
5
5
  platform: ruby
6
6
  authors:
7
7
  - Andrew Kane
8
- autorequire:
8
+ autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2018-04-10 00:00:00.000000000 Z
11
+ date: 2020-11-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: slop
@@ -16,28 +16,28 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: 4.2.0
19
+ version: 4.8.2
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - ">="
25
25
  - !ruby/object:Gem::Version
26
- version: 4.2.0
26
+ version: 4.8.2
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: pg
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - ">="
32
32
  - !ruby/object:Gem::Version
33
- version: '0'
33
+ version: 0.18.2
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
- version: '0'
40
+ version: 0.18.2
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: pg_query
43
43
  requirement: !ruby/object:Gem::Requirement
@@ -94,24 +94,17 @@ dependencies:
94
94
  - - ">="
95
95
  - !ruby/object:Gem::Version
96
96
  version: '0'
97
- description:
98
- email:
99
- - andrew@chartkick.com
97
+ description:
98
+ email: andrew@chartkick.com
100
99
  executables:
101
100
  - dexter
102
101
  extensions: []
103
102
  extra_rdoc_files: []
104
103
  files:
105
- - ".gitignore"
106
- - ".travis.yml"
107
104
  - CHANGELOG.md
108
- - Gemfile
109
105
  - LICENSE.txt
110
106
  - README.md
111
- - Rakefile
112
107
  - exe/dexter
113
- - guides/Hosted-Postgres.md
114
- - guides/Linux.md
115
108
  - lib/dexter.rb
116
109
  - lib/dexter/client.rb
117
110
  - lib/dexter/collector.rb
@@ -122,12 +115,13 @@ files:
122
115
  - lib/dexter/pg_stat_activity_parser.rb
123
116
  - lib/dexter/processor.rb
124
117
  - lib/dexter/query.rb
118
+ - lib/dexter/sql_log_parser.rb
125
119
  - lib/dexter/version.rb
126
- - pgdexter.gemspec
127
120
  homepage: https://github.com/ankane/dexter
128
- licenses: []
121
+ licenses:
122
+ - MIT
129
123
  metadata: {}
130
- post_install_message:
124
+ post_install_message:
131
125
  rdoc_options: []
132
126
  require_paths:
133
127
  - lib
@@ -135,16 +129,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
135
129
  requirements:
136
130
  - - ">="
137
131
  - !ruby/object:Gem::Version
138
- version: '0'
132
+ version: '2.2'
139
133
  required_rubygems_version: !ruby/object:Gem::Requirement
140
134
  requirements:
141
135
  - - ">="
142
136
  - !ruby/object:Gem::Version
143
137
  version: '0'
144
138
  requirements: []
145
- rubyforge_project:
146
- rubygems_version: 2.7.6
147
- signing_key:
139
+ rubygems_version: 3.1.4
140
+ signing_key:
148
141
  specification_version: 4
149
142
  summary: The automatic indexer for Postgres
150
143
  test_files: []
data/.gitignore DELETED
@@ -1,9 +0,0 @@
1
- /.bundle/
2
- /.yardoc
3
- /Gemfile.lock
4
- /_yardoc/
5
- /coverage/
6
- /doc/
7
- /pkg/
8
- /spec/reports/
9
- /tmp/
@@ -1,18 +0,0 @@
1
- language: ruby
2
- rvm: 2.4.1
3
- cache: bundler
4
- script: bundle exec rake test
5
- addons:
6
- postgresql: "9.6"
7
- before_script:
8
- - sudo apt-get install postgresql-server-dev-9.6
9
- - git clone https://github.com/HypoPG/hypopg.git
10
- - cd hypopg
11
- - git checkout 1.1.1
12
- - make
13
- - sudo make install
14
- - psql -c 'create database dexter_test;' -U postgres
15
- notifications:
16
- email:
17
- on_success: never
18
- on_failure: change
data/Gemfile DELETED
@@ -1,4 +0,0 @@
1
- source "https://rubygems.org"
2
-
3
- # Specify your gem's dependencies in dexter.gemspec
4
- gemspec
data/Rakefile DELETED
@@ -1,11 +0,0 @@
1
- require "bundler/gem_tasks"
2
- require "rake/testtask"
3
-
4
- Rake::TestTask.new(:test) do |t|
5
- t.libs << "test"
6
- t.libs << "lib"
7
- t.test_files = FileList["test/**/*_test.rb"]
8
- t.warning = false
9
- end
10
-
11
- task default: :test
@@ -1,102 +0,0 @@
1
- # Hosted Postgres
2
-
3
- Some hosted providers like Amazon RDS and Heroku do not support the HypoPG extension, which Dexter needs to run. Hopefully this will change with time. For now, we can spin up a separate database instance to run Dexter. It’s not super convenient, but can be useful to do from time to time.
4
-
5
- ### Install Postgres and Ruby
6
-
7
- Linux
8
-
9
- ```sh
10
- sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
11
- sudo apt-get install -y wget ca-certificates
12
- wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
13
- sudo apt-get update
14
- sudo apt-get install -y postgresql-9.6 postgresql-server-dev-9.6
15
- sudo -u postgres createuser $(whoami) -s
16
- sudo apt-get install -y ruby2.2 ruby2.2-dev
17
- ```
18
-
19
- Mac
20
-
21
- ```sh
22
- brew install postgresql
23
- brew install ruby
24
- ```
25
-
26
- ### Install HypoPG and Dexter
27
-
28
- HypoPG
29
-
30
- ```sh
31
- cd /tmp
32
- curl -L https://github.com/dalibo/hypopg/archive/1.0.0.tar.gz | tar xz
33
- cd hypopg-1.0.0
34
- make
35
- make install # may need sudo
36
- ```
37
-
38
- Dexter
39
-
40
- ```sh
41
- gem install pgdexter # may need sudo
42
- ```
43
-
44
- ### Download logs
45
-
46
- #### Amazon RDS
47
-
48
- Create an IAM user with the policy below:
49
-
50
- ```
51
- {
52
- "Statement": [
53
- {
54
- "Action": [
55
- "rds:DescribeDBLogFiles",
56
- "rds:DownloadDBLogFilePortion"
57
- ],
58
- "Effect": "Allow",
59
- "Resource": "*"
60
- }
61
- ]
62
- }
63
- ```
64
-
65
- And run:
66
-
67
- ```sh
68
- aws configure
69
- gem install pghero_logs # may need sudo
70
- pghero_logs download <instance-id>
71
- ```
72
-
73
- #### Heroku
74
-
75
- Production-tier databases only
76
-
77
- ```sh
78
- heroku logs -p postgres > postgresql.log
79
- ```
80
-
81
- ### Dump and restore
82
-
83
- We recommend creating a new instance from a snapshot for the dump to avoid affecting customers.
84
-
85
- ```sh
86
- pg_dump -v -j 8 -Fd -f /tmp/newout.dir <connection-options>
87
- ```
88
-
89
- Then shutdown the dump instance. Restore with:
90
-
91
- ```sh
92
- createdb dexter_restore
93
- pg_restore -v -j 8 -x -O --format=d -d dexter_restore /tmp/newout.dir/
94
- ```
95
-
96
- ### Run Dexter
97
-
98
- ```sh
99
- dexter dexter_restore postgresql.log* --analyze
100
- ```
101
-
102
- :tada:
@@ -1,72 +0,0 @@
1
- # Linux Packages
2
-
3
- Distributions
4
-
5
- - [Ubuntu 16.04 (Xenial)](#ubuntu-1604-xenial)
6
- - [Ubuntu 14.04 (Trusty)](#ubuntu-1404-trusty)
7
- - [Debian 9 (Stretch)](#debian-9-stretch)
8
- - [Debian 8 (Jesse)](#debian-8-jesse)
9
- - [CentOS / RHEL 7](#centos--rhel-7)
10
- - [SUSE Linux Enterprise Server 12](#suse-linux-enterprise-server-12)
11
-
12
- ### Ubuntu 16.04 (Xenial)
13
-
14
- ```sh
15
- wget -qO- https://dl.packager.io/srv/pghero/dexter/key | sudo apt-key add -
16
- sudo wget -O /etc/apt/sources.list.d/dexter.list \
17
- https://dl.packager.io/srv/pghero/dexter/master/installer/ubuntu/16.04.repo
18
- sudo apt-get update
19
- sudo apt-get -y install dexter
20
- ```
21
-
22
- ### Ubuntu 14.04 (Trusty)
23
-
24
- ```sh
25
- wget -qO- https://dl.packager.io/srv/pghero/dexter/key | sudo apt-key add -
26
- sudo wget -O /etc/apt/sources.list.d/dexter.list \
27
- https://dl.packager.io/srv/pghero/dexter/master/installer/ubuntu/14.04.repo
28
- sudo apt-get update
29
- sudo apt-get install dexter
30
- ```
31
-
32
- ### Debian 9 (Stretch)
33
-
34
- ```sh
35
- sudo apt-get -y install apt-transport-https
36
- wget -qO- https://dl.packager.io/srv/pghero/dexter/key | sudo apt-key add -
37
- sudo wget -O /etc/apt/sources.list.d/dexter.list \
38
- https://dl.packager.io/srv/pghero/dexter/master/installer/debian/9.repo
39
- sudo apt-get update
40
- sudo apt-get install dexter
41
- ```
42
-
43
- ### Debian 8 (Jesse)
44
-
45
- ```sh
46
- sudo apt-get -y install apt-transport-https
47
- wget -qO- https://dl.packager.io/srv/pghero/dexter/key | sudo apt-key add -
48
- sudo wget -O /etc/apt/sources.list.d/dexter.list \
49
- https://dl.packager.io/srv/pghero/dexter/master/installer/debian/8.repo
50
- sudo apt-get update
51
- sudo apt-get install dexter
52
- ```
53
-
54
- ### CentOS / RHEL 7
55
-
56
- ```sh
57
- sudo wget -O /etc/yum.repos.d/dexter.repo \
58
- https://dl.packager.io/srv/pghero/dexter/master/installer/el/7.repo
59
- sudo yum install dexter
60
- ```
61
-
62
- ### SUSE Linux Enterprise Server 12
63
-
64
- ```sh
65
- sudo wget -O /etc/zypp/repos.d/dexter.repo \
66
- https://dl.packager.io/srv/pghero/dexter/master/installer/sles/12.repo
67
- sudo zypper install dexter
68
- ```
69
-
70
- ## Credits
71
-
72
- :heart: Made possible by [Packager](https://packager.io/)
@@ -1,30 +0,0 @@
1
- # coding: utf-8
2
-
3
- lib = File.expand_path("../lib", __FILE__)
4
- $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
- require "dexter/version"
6
-
7
- Gem::Specification.new do |spec|
8
- spec.name = "pgdexter"
9
- spec.version = Dexter::VERSION
10
- spec.authors = ["Andrew Kane"]
11
- spec.email = ["andrew@chartkick.com"]
12
-
13
- spec.summary = "The automatic indexer for Postgres"
14
- spec.homepage = "https://github.com/ankane/dexter"
15
-
16
- spec.files = `git ls-files -z`.split("\x0").reject do |f|
17
- f.match(%r{^(test|spec|features)/})
18
- end
19
- spec.bindir = "exe"
20
- spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
21
- spec.require_paths = ["lib"]
22
-
23
- spec.add_dependency "slop", ">= 4.2.0"
24
- spec.add_dependency "pg"
25
- spec.add_dependency "pg_query"
26
-
27
- spec.add_development_dependency "bundler"
28
- spec.add_development_dependency "rake"
29
- spec.add_development_dependency "minitest"
30
- end