pgdexter 0.4.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8e8097505929a24e8c038c7fe69ac667faaa05008a3aa94971501fc4b9f15157
4
- data.tar.gz: a08bf061493ed984103ce768af1347d81250c51b4f068f69d3688a1e4fd25514
3
+ metadata.gz: d2b2528c0ff8201ce676b1a4ca7ec11de52013bfc608fab1b574122a31dd5ae9
4
+ data.tar.gz: 570f24c171e02425a5c492068ccedf48e318194333d825cc384ce6cfeeb6c30a
5
5
  SHA512:
6
- metadata.gz: 9c2d99ecbd5fd68e460f25982fe31d1149d9682a7065cfb674d24785f78138ab04d94bcdbf3b81f2d8065468f6531bbaea1e0593bd8fd7ab4dcdb2b061276c1e
7
- data.tar.gz: 0e55cc589470760d317e3ba0e895d7ce5e24cd79517cf02c59aaba1caf36eb1f9785725eedc61e7402d480a61870828aaf2b33683f8ca58223f380d518c7df9c
6
+ metadata.gz: e0975c42916c22b0d8f0b12f0144865370372771526ce32574a4602918aadd786b8ab39cb092ad28590c61c86820c51dcac4f7a41cc5930412868ce2a0590498
7
+ data.tar.gz: e91d272a0d6bc737634615118a1a33cbbf2ac2fe957aabb17edae9ae42fa17f596052e83a9caa80b71effa6b20316b12e551bd8aea7734648b3210bdb2bf00ec
data/CHANGELOG.md CHANGED
@@ -1,3 +1,9 @@
1
+ ## 0.4.3 (2023-03-26)
2
+
3
+ - Added experimental `--log-table` option
4
+ - Improved help
5
+ - Require pg_query < 4
6
+
1
7
  ## 0.4.2 (2023-01-29)
2
8
 
3
9
  - Fixed `--pg-stat-statements` option for Postgres 13+
data/LICENSE.txt CHANGED
@@ -1,4 +1,4 @@
1
- Copyright (c) 2017-2021 Andrew Kane
1
+ Copyright (c) 2017-2023 Andrew Kane
2
2
 
3
3
  MIT License
4
4
 
data/README.md CHANGED
@@ -110,14 +110,6 @@ or collect running queries with:
110
110
  dexter <connection-options> --pg-stat-activity
111
111
  ```
112
112
 
113
- or use the [pg_stat_statements](https://www.postgresql.org/docs/current/static/pgstatstatements.html) extension:
114
-
115
- ```sh
116
- dexter <connection-options> --pg-stat-statements
117
- ```
118
-
119
- > Note: Logs or running queries are highly preferred over pg_stat_statements, as pg_stat_statements often doesn’t store enough information to optimize queries.
120
-
121
113
  ### Collection Options
122
114
 
123
115
  To prevent one-off queries from being indexed, specify a minimum number of calls before a query is considered for indexing
@@ -214,7 +206,7 @@ And run it with:
214
206
  docker run -ti ankane/dexter <connection-options>
215
207
  ```
216
208
 
217
- For databases on the host machine, use `host.docker.internal` as the hostname (on Linux, this requires Docker 20.04 and `--add-host=host.docker.internal:host-gateway`).
209
+ For databases on the host machine, use `host.docker.internal` as the hostname (on Linux, this requires Docker 20.04+ and `--add-host=host.docker.internal:host-gateway`).
218
210
 
219
211
  ### Homebrew
220
212
 
data/lib/dexter/client.rb CHANGED
@@ -27,6 +27,8 @@ module Dexter
27
27
  Indexer.new(options).process_stat_statements
28
28
  elsif options[:pg_stat_activity]
29
29
  Processor.new(:pg_stat_activity, options).perform
30
+ elsif options[:log_table]
31
+ Processor.new(:log_table, options).perform
30
32
  elsif arguments.any?
31
33
  ARGV.replace(arguments)
32
34
  Processor.new(ARGF, options).perform
@@ -40,23 +42,45 @@ module Dexter
40
42
  o.banner = %(Usage:
41
43
  dexter [options])
42
44
  o.separator ""
43
- o.separator "Options:"
45
+
46
+ o.separator "Input options:"
47
+ o.string "--input-format", "input format", default: "stderr"
48
+ o.string "--log-table", "log table (experimental)"
49
+ o.boolean "--pg-stat-activity", "use pg_stat_activity", default: false, help: false
50
+ o.boolean "--pg-stat-statements", "use pg_stat_statements", default: false, help: false
51
+ o.string "-s", "--statement", "process a single statement"
52
+ o.separator ""
53
+
54
+ o.separator "Connection options:"
55
+ o.string "-d", "--dbname", "database name"
56
+ o.string "-h", "--host", "database host"
57
+ o.integer "-p", "--port", "database port"
58
+ o.string "-U", "--username", "database user"
59
+ o.separator ""
60
+
61
+ o.separator "Processing options:"
62
+ o.integer "--interval", "time to wait between processing queries, in seconds", default: 60
63
+ o.float "--min-calls", "only process queries that have been called a certain number of times", default: 0
64
+ o.float "--min-time", "only process queries that have consumed a certain amount of DB time, in minutes", default: 0
65
+ o.boolean "--once", "run once", default: false, help: false
66
+ o.separator ""
67
+
68
+ o.separator "Indexing options:"
44
69
  o.boolean "--analyze", "analyze tables that haven't been analyzed in the past hour", default: false
45
70
  o.boolean "--create", "create indexes", default: false
46
71
  o.array "--exclude", "prevent specific tables from being indexed"
47
72
  o.string "--include", "only include specific tables"
48
- o.string "--input-format", "input format", default: "stderr"
49
- o.integer "--interval", "time to wait between processing queries, in seconds", default: 60
73
+ o.integer "--min-cost-savings-pct", default: 50, help: false
74
+ o.string "--tablespace", "tablespace to create indexes"
75
+ o.separator ""
76
+
77
+ o.separator "Logging options:"
50
78
  o.boolean "--log-explain", "log explain", default: false, help: false
51
79
  o.string "--log-level", "log level", default: "info"
52
80
  o.boolean "--log-sql", "log sql", default: false
53
- o.float "--min-calls", "only process queries that have been called a certain number of times", default: 0
54
- o.float "--min-time", "only process queries that have consumed a certain amount of DB time, in minutes", default: 0
55
- o.integer "--min-cost-savings-pct", default: 50, help: false
56
- o.boolean "--pg-stat-activity", "use pg_stat_activity", default: false, help: false
57
- o.boolean "--pg-stat-statements", "use pg_stat_statements", default: false, help: false
58
- o.string "-s", "--statement", "process a single statement"
59
- o.string "--tablespace", "tablespace to create indexes"
81
+ o.separator ""
82
+
83
+ o.separator "Other options:"
60
84
  o.on "-v", "--version", "print the version" do
61
85
  log Dexter::VERSION
62
86
  exit
@@ -65,12 +89,6 @@ module Dexter
65
89
  log o
66
90
  exit
67
91
  end
68
- o.separator ""
69
- o.separator "Connection options:"
70
- o.string "-d", "--dbname", "database name"
71
- o.string "-h", "--host", "database host"
72
- o.integer "-p", "--port", "database port"
73
- o.string "-U", "--username", "database user"
74
92
  end
75
93
 
76
94
  arguments = opts.arguments
@@ -1,22 +1,24 @@
1
- require "csv"
2
-
3
1
  module Dexter
4
2
  class CsvLogParser < LogParser
5
3
  FIRST_LINE_REGEX = /\A.+/
6
4
 
7
5
  def perform
8
6
  CSV.new(@logfile.to_io).each do |row|
9
- if (m = REGEX.match(row[13]))
10
- # replace first line with match
11
- # needed for multiline queries
12
- active_line = row[13].sub(FIRST_LINE_REGEX, m[3])
13
-
14
- add_parameters(active_line, row[14]) if row[14]
15
- process_entry(active_line, m[1].to_f)
16
- end
7
+ process_csv_row(row[13], row[14])
17
8
  end
18
9
  rescue CSV::MalformedCSVError => e
19
10
  raise Dexter::Abort, "ERROR: #{e.message}"
20
11
  end
12
+
13
+ def process_csv_row(message, detail)
14
+ if (m = REGEX.match(message))
15
+ # replace first line with match
16
+ # needed for multiline queries
17
+ active_line = message.sub(FIRST_LINE_REGEX, m[3])
18
+
19
+ add_parameters(active_line, detail) if detail
20
+ process_entry(active_line, m[1].to_f)
21
+ end
22
+ end
21
23
  end
22
24
  end
@@ -0,0 +1,20 @@
1
+ module Dexter
2
+ class CsvLogTableParser < CsvLogParser
3
+ def perform
4
+ last_log_time = Time.at(0).iso8601(3)
5
+
6
+ loop do
7
+ @logfile.csvlog_activity(last_log_time).each do |row|
8
+ process_csv_row(row["message"], row["detail"])
9
+ last_log_time = row["log_time"]
10
+ end
11
+
12
+ break
13
+
14
+ # possibly enable later
15
+ # break if once
16
+ # sleep(1)
17
+ end
18
+ end
19
+ end
20
+ end
@@ -14,6 +14,7 @@ module Dexter
14
14
  @min_calls = options[:min_calls] || 0
15
15
  @analyze = options[:analyze]
16
16
  @min_cost_savings_pct = options[:min_cost_savings_pct].to_i
17
+ @log_table = options[:log_table]
17
18
  @options = options
18
19
  @mutex = Mutex.new
19
20
 
@@ -28,7 +29,7 @@ module Dexter
28
29
  end
29
30
 
30
31
  def stat_activity
31
- execute <<-SQL
32
+ execute <<~SQL
32
33
  SELECT
33
34
  pid || ':' || COALESCE(query_start, xact_start) AS id,
34
35
  query,
@@ -44,6 +45,36 @@ module Dexter
44
45
  SQL
45
46
  end
46
47
 
48
+ # works with
49
+ # file_fdw: https://www.postgresql.org/docs/current/file-fdw.html
50
+ # log_fdw: https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.PostgreSQL.CommonDBATasks.Extensions.foreign-data-wrappers.html
51
+ def csvlog_activity(last_log_time)
52
+ query = <<~SQL
53
+ SELECT
54
+ log_time,
55
+ message,
56
+ detail
57
+ FROM
58
+ #{conn.quote_ident(@log_table)}
59
+ WHERE
60
+ log_time >= \$1
61
+ SQL
62
+ execute(query, params: [last_log_time])
63
+ end
64
+
65
+ # works with
66
+ # file_fdw: https://www.postgresql.org/docs/current/file-fdw.html
67
+ # log_fdw: https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.PostgreSQL.CommonDBATasks.Extensions.foreign-data-wrappers.html
68
+ def stderr_activity
69
+ query = <<~SQL
70
+ SELECT
71
+ log_entry
72
+ FROM
73
+ #{conn.quote_ident(@log_table)}
74
+ SQL
75
+ execute(query)
76
+ end
77
+
47
78
  def process_queries(queries)
48
79
  # reset hypothetical indexes
49
80
  reset_hypothetical_indexes
@@ -141,7 +172,7 @@ module Dexter
141
172
  def analyze_tables(tables)
142
173
  tables = tables.to_a.sort
143
174
 
144
- analyze_stats = execute <<-SQL
175
+ analyze_stats = execute <<~SQL
145
176
  SELECT
146
177
  schemaname || '.' || relname AS table,
147
178
  last_analyze,
@@ -529,7 +560,7 @@ module Dexter
529
560
  raise Dexter::Abort, e.message
530
561
  end
531
562
 
532
- def execute(query, pretty: true)
563
+ def execute(query, pretty: true, params: [])
533
564
  # use exec_params instead of exec for security
534
565
  #
535
566
  # Unlike PQexec, PQexecParams allows at most one SQL command in the given string.
@@ -541,7 +572,7 @@ module Dexter
541
572
  log colorize("[sql] #{query}", :cyan) if @log_sql
542
573
 
543
574
  @mutex.synchronize do
544
- conn.exec_params(query, []).to_a
575
+ conn.exec_params(query, params).to_a
545
576
  end
546
577
  end
547
578
 
@@ -565,7 +596,7 @@ module Dexter
565
596
  end
566
597
 
567
598
  def database_tables
568
- result = execute <<-SQL
599
+ result = execute <<~SQL
569
600
  SELECT
570
601
  table_schema || '.' || table_name AS table_name
571
602
  FROM
@@ -578,7 +609,7 @@ module Dexter
578
609
 
579
610
  def materialized_views
580
611
  if server_version_num >= 90300
581
- result = execute <<-SQL
612
+ result = execute <<~SQL
582
613
  SELECT
583
614
  schemaname || '.' || matviewname AS table_name
584
615
  FROM
@@ -595,7 +626,7 @@ module Dexter
595
626
  end
596
627
 
597
628
  def database_view_tables
598
- result = execute <<-SQL
629
+ result = execute <<~SQL
599
630
  SELECT
600
631
  schemaname || '.' || viewname AS table_name,
601
632
  definition
@@ -621,7 +652,7 @@ module Dexter
621
652
 
622
653
  def stat_statements
623
654
  total_time = server_version_num >= 130000 ? "(total_plan_time + total_exec_time)" : "total_time"
624
- result = execute <<-SQL
655
+ result = execute <<~SQL
625
656
  SELECT
626
657
  DISTINCT query
627
658
  FROM
@@ -667,7 +698,7 @@ module Dexter
667
698
  end
668
699
 
669
700
  def columns(tables)
670
- columns = execute <<-SQL
701
+ columns = execute <<~SQL
671
702
  SELECT
672
703
  s.nspname || '.' || t.relname AS table_name,
673
704
  a.attname AS column_name,
@@ -686,7 +717,7 @@ module Dexter
686
717
  end
687
718
 
688
719
  def indexes(tables)
689
- execute(<<-SQL
720
+ query = <<~SQL
690
721
  SELECT
691
722
  schemaname || '.' || t.relname AS table,
692
723
  ix.relname AS name,
@@ -708,7 +739,7 @@ module Dexter
708
739
  ORDER BY
709
740
  1, 2
710
741
  SQL
711
- ).map { |v| v["columns"] = v["columns"].sub(") WHERE (", " WHERE ").split(", ").map { |c| unquote(c) }; v }
742
+ execute(query).map { |v| v["columns"] = v["columns"].sub(") WHERE (", " WHERE ").split(", ").map { |c| unquote(c) }; v }
712
743
  end
713
744
 
714
745
  def search_path
@@ -1,5 +1,3 @@
1
- require "json"
2
-
3
1
  module Dexter
4
2
  class JsonLogParser < LogParser
5
3
  FIRST_LINE_REGEX = /\A.+/
@@ -6,35 +6,13 @@ module Dexter
6
6
  LINE_SEPERATOR = ": ".freeze
7
7
  DETAIL_LINE = "DETAIL: ".freeze
8
8
 
9
+ attr_accessor :once
10
+
9
11
  def initialize(logfile, collector)
10
12
  @logfile = logfile
11
13
  @collector = collector
12
14
  end
13
15
 
14
- def perform
15
- active_line = nil
16
- duration = nil
17
-
18
- @logfile.each_line do |line|
19
- if active_line
20
- if line.include?(DETAIL_LINE)
21
- add_parameters(active_line, line.chomp.split(DETAIL_LINE)[1])
22
- elsif line.include?(LINE_SEPERATOR)
23
- process_entry(active_line, duration)
24
- active_line = nil
25
- else
26
- active_line << line
27
- end
28
- end
29
-
30
- if !active_line && (m = REGEX.match(line.chomp))
31
- duration = m[1].to_f
32
- active_line = m[3]
33
- end
34
- end
35
- process_entry(active_line, duration) if active_line
36
- end
37
-
38
16
  private
39
17
 
40
18
  def process_entry(query, duration)
@@ -18,6 +18,8 @@ module Dexter
18
18
 
19
19
  queries = new_queries
20
20
 
21
+ break if once
22
+
21
23
  sleep(1)
22
24
  end
23
25
  end
@@ -11,6 +11,12 @@ module Dexter
11
11
  @log_parser =
12
12
  if @logfile == :pg_stat_activity
13
13
  PgStatActivityParser.new(@indexer, @collector)
14
+ elsif @logfile == :log_table
15
+ if options[:input_format] == "csv"
16
+ CsvLogTableParser.new(@indexer, @collector)
17
+ else
18
+ StderrLogTableParser.new(@indexer, @collector)
19
+ end
14
20
  elsif options[:input_format] == "csv"
15
21
  CsvLogParser.new(logfile, @collector)
16
22
  elsif options[:input_format] == "json"
@@ -18,11 +24,12 @@ module Dexter
18
24
  elsif options[:input_format] == "sql"
19
25
  SqlLogParser.new(logfile, @collector)
20
26
  else
21
- LogParser.new(logfile, @collector)
27
+ StderrLogParser.new(logfile, @collector)
22
28
  end
23
29
 
24
30
  @starting_interval = 3
25
31
  @interval = options[:interval]
32
+ @log_parser.once = options[:once]
26
33
 
27
34
  @mutex = Mutex.new
28
35
  @last_checked_at = {}
@@ -31,7 +38,7 @@ module Dexter
31
38
  end
32
39
 
33
40
  def perform
34
- if [STDIN, :pg_stat_activity].include?(@logfile)
41
+ if [STDIN, :pg_stat_activity, :log_table].include?(@logfile) && !@log_parser.once
35
42
  Thread.abort_on_exception = true
36
43
  Thread.new do
37
44
  sleep(@starting_interval)
@@ -0,0 +1,31 @@
1
+ module Dexter
2
+ class StderrLogParser < LogParser
3
+ def perform
4
+ process_stderr(@logfile.each_line)
5
+ end
6
+
7
+ def process_stderr(rows)
8
+ active_line = nil
9
+ duration = nil
10
+
11
+ rows.each do |line|
12
+ if active_line
13
+ if line.include?(DETAIL_LINE)
14
+ add_parameters(active_line, line.chomp.split(DETAIL_LINE)[1])
15
+ elsif line.include?(LINE_SEPERATOR)
16
+ process_entry(active_line, duration)
17
+ active_line = nil
18
+ else
19
+ active_line << line
20
+ end
21
+ end
22
+
23
+ if !active_line && (m = REGEX.match(line.chomp))
24
+ duration = m[1].to_f
25
+ active_line = m[3]
26
+ end
27
+ end
28
+ process_entry(active_line, duration) if active_line
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,7 @@
1
+ module Dexter
2
+ class StderrLogTableParser < StderrLogParser
3
+ def perform
4
+ process_stderr(@logfile.stderr_activity.map { |r| r["log_entry"] })
5
+ end
6
+ end
7
+ end
@@ -1,3 +1,3 @@
1
1
  module Dexter
2
- VERSION = "0.4.2"
2
+ VERSION = "0.4.3"
3
3
  end
data/lib/dexter.rb CHANGED
@@ -4,23 +4,29 @@ require "pg_query"
4
4
  require "slop"
5
5
 
6
6
  # stdlib
7
+ require "csv"
7
8
  require "json"
8
9
  require "set"
9
10
  require "time"
10
11
 
11
12
  # modules
12
- require "dexter/version"
13
- require "dexter/logging"
14
- require "dexter/client"
15
- require "dexter/collector"
16
- require "dexter/indexer"
17
- require "dexter/log_parser"
18
- require "dexter/csv_log_parser"
19
- require "dexter/json_log_parser"
20
- require "dexter/pg_stat_activity_parser"
21
- require "dexter/sql_log_parser"
22
- require "dexter/processor"
23
- require "dexter/query"
13
+ require_relative "dexter/logging"
14
+ require_relative "dexter/client"
15
+ require_relative "dexter/collector"
16
+ require_relative "dexter/indexer"
17
+ require_relative "dexter/processor"
18
+ require_relative "dexter/query"
19
+ require_relative "dexter/version"
20
+
21
+ # parsers
22
+ require_relative "dexter/log_parser"
23
+ require_relative "dexter/csv_log_parser"
24
+ require_relative "dexter/csv_log_table_parser"
25
+ require_relative "dexter/json_log_parser"
26
+ require_relative "dexter/pg_stat_activity_parser"
27
+ require_relative "dexter/sql_log_parser"
28
+ require_relative "dexter/stderr_log_parser"
29
+ require_relative "dexter/stderr_log_table_parser"
24
30
 
25
31
  module Dexter
26
32
  class Abort < StandardError; end
metadata CHANGED
@@ -1,57 +1,57 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pgdexter
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.2
4
+ version: 0.4.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Andrew Kane
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2023-01-30 00:00:00.000000000 Z
11
+ date: 2023-03-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: slop
14
+ name: pg
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: 4.8.2
19
+ version: 0.18.2
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - ">="
25
25
  - !ruby/object:Gem::Version
26
- version: 4.8.2
26
+ version: 0.18.2
27
27
  - !ruby/object:Gem::Dependency
28
- name: pg
28
+ name: pg_query
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - ">="
31
+ - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: 0.18.2
33
+ version: '2.1'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - ">="
38
+ - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: 0.18.2
40
+ version: '2.1'
41
41
  - !ruby/object:Gem::Dependency
42
- name: pg_query
42
+ name: slop
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
45
  - - ">="
46
46
  - !ruby/object:Gem::Version
47
- version: '2.1'
47
+ version: 4.10.1
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - ">="
53
53
  - !ruby/object:Gem::Version
54
- version: '2.1'
54
+ version: 4.10.1
55
55
  description:
56
56
  email: andrew@ankane.org
57
57
  executables:
@@ -67,6 +67,7 @@ files:
67
67
  - lib/dexter/client.rb
68
68
  - lib/dexter/collector.rb
69
69
  - lib/dexter/csv_log_parser.rb
70
+ - lib/dexter/csv_log_table_parser.rb
70
71
  - lib/dexter/indexer.rb
71
72
  - lib/dexter/json_log_parser.rb
72
73
  - lib/dexter/log_parser.rb
@@ -75,6 +76,8 @@ files:
75
76
  - lib/dexter/processor.rb
76
77
  - lib/dexter/query.rb
77
78
  - lib/dexter/sql_log_parser.rb
79
+ - lib/dexter/stderr_log_parser.rb
80
+ - lib/dexter/stderr_log_table_parser.rb
78
81
  - lib/dexter/version.rb
79
82
  homepage: https://github.com/ankane/dexter
80
83
  licenses:
@@ -95,7 +98,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
95
98
  - !ruby/object:Gem::Version
96
99
  version: '0'
97
100
  requirements: []
98
- rubygems_version: 3.4.1
101
+ rubygems_version: 3.4.6
99
102
  signing_key:
100
103
  specification_version: 4
101
104
  summary: The automatic indexer for Postgres