pgdexter 0.4.2 → 0.4.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8e8097505929a24e8c038c7fe69ac667faaa05008a3aa94971501fc4b9f15157
4
- data.tar.gz: a08bf061493ed984103ce768af1347d81250c51b4f068f69d3688a1e4fd25514
3
+ metadata.gz: d2b2528c0ff8201ce676b1a4ca7ec11de52013bfc608fab1b574122a31dd5ae9
4
+ data.tar.gz: 570f24c171e02425a5c492068ccedf48e318194333d825cc384ce6cfeeb6c30a
5
5
  SHA512:
6
- metadata.gz: 9c2d99ecbd5fd68e460f25982fe31d1149d9682a7065cfb674d24785f78138ab04d94bcdbf3b81f2d8065468f6531bbaea1e0593bd8fd7ab4dcdb2b061276c1e
7
- data.tar.gz: 0e55cc589470760d317e3ba0e895d7ce5e24cd79517cf02c59aaba1caf36eb1f9785725eedc61e7402d480a61870828aaf2b33683f8ca58223f380d518c7df9c
6
+ metadata.gz: e0975c42916c22b0d8f0b12f0144865370372771526ce32574a4602918aadd786b8ab39cb092ad28590c61c86820c51dcac4f7a41cc5930412868ce2a0590498
7
+ data.tar.gz: e91d272a0d6bc737634615118a1a33cbbf2ac2fe957aabb17edae9ae42fa17f596052e83a9caa80b71effa6b20316b12e551bd8aea7734648b3210bdb2bf00ec
data/CHANGELOG.md CHANGED
@@ -1,3 +1,9 @@
1
+ ## 0.4.3 (2023-03-26)
2
+
3
+ - Added experimental `--log-table` option
4
+ - Improved help
5
+ - Require pg_query < 4
6
+
1
7
  ## 0.4.2 (2023-01-29)
2
8
 
3
9
  - Fixed `--pg-stat-statements` option for Postgres 13+
data/LICENSE.txt CHANGED
@@ -1,4 +1,4 @@
1
- Copyright (c) 2017-2021 Andrew Kane
1
+ Copyright (c) 2017-2023 Andrew Kane
2
2
 
3
3
  MIT License
4
4
 
data/README.md CHANGED
@@ -110,14 +110,6 @@ or collect running queries with:
110
110
  dexter <connection-options> --pg-stat-activity
111
111
  ```
112
112
 
113
- or use the [pg_stat_statements](https://www.postgresql.org/docs/current/static/pgstatstatements.html) extension:
114
-
115
- ```sh
116
- dexter <connection-options> --pg-stat-statements
117
- ```
118
-
119
- > Note: Logs or running queries are highly preferred over pg_stat_statements, as pg_stat_statements often doesn’t store enough information to optimize queries.
120
-
121
113
  ### Collection Options
122
114
 
123
115
  To prevent one-off queries from being indexed, specify a minimum number of calls before a query is considered for indexing
@@ -214,7 +206,7 @@ And run it with:
214
206
  docker run -ti ankane/dexter <connection-options>
215
207
  ```
216
208
 
217
- For databases on the host machine, use `host.docker.internal` as the hostname (on Linux, this requires Docker 20.04 and `--add-host=host.docker.internal:host-gateway`).
209
+ For databases on the host machine, use `host.docker.internal` as the hostname (on Linux, this requires Docker 20.04+ and `--add-host=host.docker.internal:host-gateway`).
218
210
 
219
211
  ### Homebrew
220
212
 
data/lib/dexter/client.rb CHANGED
@@ -27,6 +27,8 @@ module Dexter
27
27
  Indexer.new(options).process_stat_statements
28
28
  elsif options[:pg_stat_activity]
29
29
  Processor.new(:pg_stat_activity, options).perform
30
+ elsif options[:log_table]
31
+ Processor.new(:log_table, options).perform
30
32
  elsif arguments.any?
31
33
  ARGV.replace(arguments)
32
34
  Processor.new(ARGF, options).perform
@@ -40,23 +42,45 @@ module Dexter
40
42
  o.banner = %(Usage:
41
43
  dexter [options])
42
44
  o.separator ""
43
- o.separator "Options:"
45
+
46
+ o.separator "Input options:"
47
+ o.string "--input-format", "input format", default: "stderr"
48
+ o.string "--log-table", "log table (experimental)"
49
+ o.boolean "--pg-stat-activity", "use pg_stat_activity", default: false, help: false
50
+ o.boolean "--pg-stat-statements", "use pg_stat_statements", default: false, help: false
51
+ o.string "-s", "--statement", "process a single statement"
52
+ o.separator ""
53
+
54
+ o.separator "Connection options:"
55
+ o.string "-d", "--dbname", "database name"
56
+ o.string "-h", "--host", "database host"
57
+ o.integer "-p", "--port", "database port"
58
+ o.string "-U", "--username", "database user"
59
+ o.separator ""
60
+
61
+ o.separator "Processing options:"
62
+ o.integer "--interval", "time to wait between processing queries, in seconds", default: 60
63
+ o.float "--min-calls", "only process queries that have been called a certain number of times", default: 0
64
+ o.float "--min-time", "only process queries that have consumed a certain amount of DB time, in minutes", default: 0
65
+ o.boolean "--once", "run once", default: false, help: false
66
+ o.separator ""
67
+
68
+ o.separator "Indexing options:"
44
69
  o.boolean "--analyze", "analyze tables that haven't been analyzed in the past hour", default: false
45
70
  o.boolean "--create", "create indexes", default: false
46
71
  o.array "--exclude", "prevent specific tables from being indexed"
47
72
  o.string "--include", "only include specific tables"
48
- o.string "--input-format", "input format", default: "stderr"
49
- o.integer "--interval", "time to wait between processing queries, in seconds", default: 60
73
+ o.integer "--min-cost-savings-pct", default: 50, help: false
74
+ o.string "--tablespace", "tablespace to create indexes"
75
+ o.separator ""
76
+
77
+ o.separator "Logging options:"
50
78
  o.boolean "--log-explain", "log explain", default: false, help: false
51
79
  o.string "--log-level", "log level", default: "info"
52
80
  o.boolean "--log-sql", "log sql", default: false
53
- o.float "--min-calls", "only process queries that have been called a certain number of times", default: 0
54
- o.float "--min-time", "only process queries that have consumed a certain amount of DB time, in minutes", default: 0
55
- o.integer "--min-cost-savings-pct", default: 50, help: false
56
- o.boolean "--pg-stat-activity", "use pg_stat_activity", default: false, help: false
57
- o.boolean "--pg-stat-statements", "use pg_stat_statements", default: false, help: false
58
- o.string "-s", "--statement", "process a single statement"
59
- o.string "--tablespace", "tablespace to create indexes"
81
+ o.separator ""
82
+
83
+ o.separator "Other options:"
60
84
  o.on "-v", "--version", "print the version" do
61
85
  log Dexter::VERSION
62
86
  exit
@@ -65,12 +89,6 @@ module Dexter
65
89
  log o
66
90
  exit
67
91
  end
68
- o.separator ""
69
- o.separator "Connection options:"
70
- o.string "-d", "--dbname", "database name"
71
- o.string "-h", "--host", "database host"
72
- o.integer "-p", "--port", "database port"
73
- o.string "-U", "--username", "database user"
74
92
  end
75
93
 
76
94
  arguments = opts.arguments
@@ -1,22 +1,24 @@
1
- require "csv"
2
-
3
1
  module Dexter
4
2
  class CsvLogParser < LogParser
5
3
  FIRST_LINE_REGEX = /\A.+/
6
4
 
7
5
  def perform
8
6
  CSV.new(@logfile.to_io).each do |row|
9
- if (m = REGEX.match(row[13]))
10
- # replace first line with match
11
- # needed for multiline queries
12
- active_line = row[13].sub(FIRST_LINE_REGEX, m[3])
13
-
14
- add_parameters(active_line, row[14]) if row[14]
15
- process_entry(active_line, m[1].to_f)
16
- end
7
+ process_csv_row(row[13], row[14])
17
8
  end
18
9
  rescue CSV::MalformedCSVError => e
19
10
  raise Dexter::Abort, "ERROR: #{e.message}"
20
11
  end
12
+
13
+ def process_csv_row(message, detail)
14
+ if (m = REGEX.match(message))
15
+ # replace first line with match
16
+ # needed for multiline queries
17
+ active_line = message.sub(FIRST_LINE_REGEX, m[3])
18
+
19
+ add_parameters(active_line, detail) if detail
20
+ process_entry(active_line, m[1].to_f)
21
+ end
22
+ end
21
23
  end
22
24
  end
@@ -0,0 +1,20 @@
1
+ module Dexter
2
+ class CsvLogTableParser < CsvLogParser
3
+ def perform
4
+ last_log_time = Time.at(0).iso8601(3)
5
+
6
+ loop do
7
+ @logfile.csvlog_activity(last_log_time).each do |row|
8
+ process_csv_row(row["message"], row["detail"])
9
+ last_log_time = row["log_time"]
10
+ end
11
+
12
+ break
13
+
14
+ # possibly enable later
15
+ # break if once
16
+ # sleep(1)
17
+ end
18
+ end
19
+ end
20
+ end
@@ -14,6 +14,7 @@ module Dexter
14
14
  @min_calls = options[:min_calls] || 0
15
15
  @analyze = options[:analyze]
16
16
  @min_cost_savings_pct = options[:min_cost_savings_pct].to_i
17
+ @log_table = options[:log_table]
17
18
  @options = options
18
19
  @mutex = Mutex.new
19
20
 
@@ -28,7 +29,7 @@ module Dexter
28
29
  end
29
30
 
30
31
  def stat_activity
31
- execute <<-SQL
32
+ execute <<~SQL
32
33
  SELECT
33
34
  pid || ':' || COALESCE(query_start, xact_start) AS id,
34
35
  query,
@@ -44,6 +45,36 @@ module Dexter
44
45
  SQL
45
46
  end
46
47
 
48
+ # works with
49
+ # file_fdw: https://www.postgresql.org/docs/current/file-fdw.html
50
+ # log_fdw: https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.PostgreSQL.CommonDBATasks.Extensions.foreign-data-wrappers.html
51
+ def csvlog_activity(last_log_time)
52
+ query = <<~SQL
53
+ SELECT
54
+ log_time,
55
+ message,
56
+ detail
57
+ FROM
58
+ #{conn.quote_ident(@log_table)}
59
+ WHERE
60
+ log_time >= \$1
61
+ SQL
62
+ execute(query, params: [last_log_time])
63
+ end
64
+
65
+ # works with
66
+ # file_fdw: https://www.postgresql.org/docs/current/file-fdw.html
67
+ # log_fdw: https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.PostgreSQL.CommonDBATasks.Extensions.foreign-data-wrappers.html
68
+ def stderr_activity
69
+ query = <<~SQL
70
+ SELECT
71
+ log_entry
72
+ FROM
73
+ #{conn.quote_ident(@log_table)}
74
+ SQL
75
+ execute(query)
76
+ end
77
+
47
78
  def process_queries(queries)
48
79
  # reset hypothetical indexes
49
80
  reset_hypothetical_indexes
@@ -141,7 +172,7 @@ module Dexter
141
172
  def analyze_tables(tables)
142
173
  tables = tables.to_a.sort
143
174
 
144
- analyze_stats = execute <<-SQL
175
+ analyze_stats = execute <<~SQL
145
176
  SELECT
146
177
  schemaname || '.' || relname AS table,
147
178
  last_analyze,
@@ -529,7 +560,7 @@ module Dexter
529
560
  raise Dexter::Abort, e.message
530
561
  end
531
562
 
532
- def execute(query, pretty: true)
563
+ def execute(query, pretty: true, params: [])
533
564
  # use exec_params instead of exec for security
534
565
  #
535
566
  # Unlike PQexec, PQexecParams allows at most one SQL command in the given string.
@@ -541,7 +572,7 @@ module Dexter
541
572
  log colorize("[sql] #{query}", :cyan) if @log_sql
542
573
 
543
574
  @mutex.synchronize do
544
- conn.exec_params(query, []).to_a
575
+ conn.exec_params(query, params).to_a
545
576
  end
546
577
  end
547
578
 
@@ -565,7 +596,7 @@ module Dexter
565
596
  end
566
597
 
567
598
  def database_tables
568
- result = execute <<-SQL
599
+ result = execute <<~SQL
569
600
  SELECT
570
601
  table_schema || '.' || table_name AS table_name
571
602
  FROM
@@ -578,7 +609,7 @@ module Dexter
578
609
 
579
610
  def materialized_views
580
611
  if server_version_num >= 90300
581
- result = execute <<-SQL
612
+ result = execute <<~SQL
582
613
  SELECT
583
614
  schemaname || '.' || matviewname AS table_name
584
615
  FROM
@@ -595,7 +626,7 @@ module Dexter
595
626
  end
596
627
 
597
628
  def database_view_tables
598
- result = execute <<-SQL
629
+ result = execute <<~SQL
599
630
  SELECT
600
631
  schemaname || '.' || viewname AS table_name,
601
632
  definition
@@ -621,7 +652,7 @@ module Dexter
621
652
 
622
653
  def stat_statements
623
654
  total_time = server_version_num >= 130000 ? "(total_plan_time + total_exec_time)" : "total_time"
624
- result = execute <<-SQL
655
+ result = execute <<~SQL
625
656
  SELECT
626
657
  DISTINCT query
627
658
  FROM
@@ -667,7 +698,7 @@ module Dexter
667
698
  end
668
699
 
669
700
  def columns(tables)
670
- columns = execute <<-SQL
701
+ columns = execute <<~SQL
671
702
  SELECT
672
703
  s.nspname || '.' || t.relname AS table_name,
673
704
  a.attname AS column_name,
@@ -686,7 +717,7 @@ module Dexter
686
717
  end
687
718
 
688
719
  def indexes(tables)
689
- execute(<<-SQL
720
+ query = <<~SQL
690
721
  SELECT
691
722
  schemaname || '.' || t.relname AS table,
692
723
  ix.relname AS name,
@@ -708,7 +739,7 @@ module Dexter
708
739
  ORDER BY
709
740
  1, 2
710
741
  SQL
711
- ).map { |v| v["columns"] = v["columns"].sub(") WHERE (", " WHERE ").split(", ").map { |c| unquote(c) }; v }
742
+ execute(query).map { |v| v["columns"] = v["columns"].sub(") WHERE (", " WHERE ").split(", ").map { |c| unquote(c) }; v }
712
743
  end
713
744
 
714
745
  def search_path
@@ -1,5 +1,3 @@
1
- require "json"
2
-
3
1
  module Dexter
4
2
  class JsonLogParser < LogParser
5
3
  FIRST_LINE_REGEX = /\A.+/
@@ -6,35 +6,13 @@ module Dexter
6
6
  LINE_SEPERATOR = ": ".freeze
7
7
  DETAIL_LINE = "DETAIL: ".freeze
8
8
 
9
+ attr_accessor :once
10
+
9
11
  def initialize(logfile, collector)
10
12
  @logfile = logfile
11
13
  @collector = collector
12
14
  end
13
15
 
14
- def perform
15
- active_line = nil
16
- duration = nil
17
-
18
- @logfile.each_line do |line|
19
- if active_line
20
- if line.include?(DETAIL_LINE)
21
- add_parameters(active_line, line.chomp.split(DETAIL_LINE)[1])
22
- elsif line.include?(LINE_SEPERATOR)
23
- process_entry(active_line, duration)
24
- active_line = nil
25
- else
26
- active_line << line
27
- end
28
- end
29
-
30
- if !active_line && (m = REGEX.match(line.chomp))
31
- duration = m[1].to_f
32
- active_line = m[3]
33
- end
34
- end
35
- process_entry(active_line, duration) if active_line
36
- end
37
-
38
16
  private
39
17
 
40
18
  def process_entry(query, duration)
@@ -18,6 +18,8 @@ module Dexter
18
18
 
19
19
  queries = new_queries
20
20
 
21
+ break if once
22
+
21
23
  sleep(1)
22
24
  end
23
25
  end
@@ -11,6 +11,12 @@ module Dexter
11
11
  @log_parser =
12
12
  if @logfile == :pg_stat_activity
13
13
  PgStatActivityParser.new(@indexer, @collector)
14
+ elsif @logfile == :log_table
15
+ if options[:input_format] == "csv"
16
+ CsvLogTableParser.new(@indexer, @collector)
17
+ else
18
+ StderrLogTableParser.new(@indexer, @collector)
19
+ end
14
20
  elsif options[:input_format] == "csv"
15
21
  CsvLogParser.new(logfile, @collector)
16
22
  elsif options[:input_format] == "json"
@@ -18,11 +24,12 @@ module Dexter
18
24
  elsif options[:input_format] == "sql"
19
25
  SqlLogParser.new(logfile, @collector)
20
26
  else
21
- LogParser.new(logfile, @collector)
27
+ StderrLogParser.new(logfile, @collector)
22
28
  end
23
29
 
24
30
  @starting_interval = 3
25
31
  @interval = options[:interval]
32
+ @log_parser.once = options[:once]
26
33
 
27
34
  @mutex = Mutex.new
28
35
  @last_checked_at = {}
@@ -31,7 +38,7 @@ module Dexter
31
38
  end
32
39
 
33
40
  def perform
34
- if [STDIN, :pg_stat_activity].include?(@logfile)
41
+ if [STDIN, :pg_stat_activity, :log_table].include?(@logfile) && !@log_parser.once
35
42
  Thread.abort_on_exception = true
36
43
  Thread.new do
37
44
  sleep(@starting_interval)
@@ -0,0 +1,31 @@
1
+ module Dexter
2
+ class StderrLogParser < LogParser
3
+ def perform
4
+ process_stderr(@logfile.each_line)
5
+ end
6
+
7
+ def process_stderr(rows)
8
+ active_line = nil
9
+ duration = nil
10
+
11
+ rows.each do |line|
12
+ if active_line
13
+ if line.include?(DETAIL_LINE)
14
+ add_parameters(active_line, line.chomp.split(DETAIL_LINE)[1])
15
+ elsif line.include?(LINE_SEPERATOR)
16
+ process_entry(active_line, duration)
17
+ active_line = nil
18
+ else
19
+ active_line << line
20
+ end
21
+ end
22
+
23
+ if !active_line && (m = REGEX.match(line.chomp))
24
+ duration = m[1].to_f
25
+ active_line = m[3]
26
+ end
27
+ end
28
+ process_entry(active_line, duration) if active_line
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,7 @@
1
+ module Dexter
2
+ class StderrLogTableParser < StderrLogParser
3
+ def perform
4
+ process_stderr(@logfile.stderr_activity.map { |r| r["log_entry"] })
5
+ end
6
+ end
7
+ end
@@ -1,3 +1,3 @@
1
1
  module Dexter
2
- VERSION = "0.4.2"
2
+ VERSION = "0.4.3"
3
3
  end
data/lib/dexter.rb CHANGED
@@ -4,23 +4,29 @@ require "pg_query"
4
4
  require "slop"
5
5
 
6
6
  # stdlib
7
+ require "csv"
7
8
  require "json"
8
9
  require "set"
9
10
  require "time"
10
11
 
11
12
  # modules
12
- require "dexter/version"
13
- require "dexter/logging"
14
- require "dexter/client"
15
- require "dexter/collector"
16
- require "dexter/indexer"
17
- require "dexter/log_parser"
18
- require "dexter/csv_log_parser"
19
- require "dexter/json_log_parser"
20
- require "dexter/pg_stat_activity_parser"
21
- require "dexter/sql_log_parser"
22
- require "dexter/processor"
23
- require "dexter/query"
13
+ require_relative "dexter/logging"
14
+ require_relative "dexter/client"
15
+ require_relative "dexter/collector"
16
+ require_relative "dexter/indexer"
17
+ require_relative "dexter/processor"
18
+ require_relative "dexter/query"
19
+ require_relative "dexter/version"
20
+
21
+ # parsers
22
+ require_relative "dexter/log_parser"
23
+ require_relative "dexter/csv_log_parser"
24
+ require_relative "dexter/csv_log_table_parser"
25
+ require_relative "dexter/json_log_parser"
26
+ require_relative "dexter/pg_stat_activity_parser"
27
+ require_relative "dexter/sql_log_parser"
28
+ require_relative "dexter/stderr_log_parser"
29
+ require_relative "dexter/stderr_log_table_parser"
24
30
 
25
31
  module Dexter
26
32
  class Abort < StandardError; end
metadata CHANGED
@@ -1,57 +1,57 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pgdexter
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.2
4
+ version: 0.4.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Andrew Kane
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2023-01-30 00:00:00.000000000 Z
11
+ date: 2023-03-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: slop
14
+ name: pg
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: 4.8.2
19
+ version: 0.18.2
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - ">="
25
25
  - !ruby/object:Gem::Version
26
- version: 4.8.2
26
+ version: 0.18.2
27
27
  - !ruby/object:Gem::Dependency
28
- name: pg
28
+ name: pg_query
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - ">="
31
+ - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: 0.18.2
33
+ version: '2.1'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - ">="
38
+ - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: 0.18.2
40
+ version: '2.1'
41
41
  - !ruby/object:Gem::Dependency
42
- name: pg_query
42
+ name: slop
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
45
  - - ">="
46
46
  - !ruby/object:Gem::Version
47
- version: '2.1'
47
+ version: 4.10.1
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - ">="
53
53
  - !ruby/object:Gem::Version
54
- version: '2.1'
54
+ version: 4.10.1
55
55
  description:
56
56
  email: andrew@ankane.org
57
57
  executables:
@@ -67,6 +67,7 @@ files:
67
67
  - lib/dexter/client.rb
68
68
  - lib/dexter/collector.rb
69
69
  - lib/dexter/csv_log_parser.rb
70
+ - lib/dexter/csv_log_table_parser.rb
70
71
  - lib/dexter/indexer.rb
71
72
  - lib/dexter/json_log_parser.rb
72
73
  - lib/dexter/log_parser.rb
@@ -75,6 +76,8 @@ files:
75
76
  - lib/dexter/processor.rb
76
77
  - lib/dexter/query.rb
77
78
  - lib/dexter/sql_log_parser.rb
79
+ - lib/dexter/stderr_log_parser.rb
80
+ - lib/dexter/stderr_log_table_parser.rb
78
81
  - lib/dexter/version.rb
79
82
  homepage: https://github.com/ankane/dexter
80
83
  licenses:
@@ -95,7 +98,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
95
98
  - !ruby/object:Gem::Version
96
99
  version: '0'
97
100
  requirements: []
98
- rubygems_version: 3.4.1
101
+ rubygems_version: 3.4.6
99
102
  signing_key:
100
103
  specification_version: 4
101
104
  summary: The automatic indexer for Postgres