fluent-plugin-perf-tools 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (98) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +15 -0
  3. data/.rubocop.yml +26 -0
  4. data/.ruby-version +1 -0
  5. data/CHANGELOG.md +5 -0
  6. data/CODE_OF_CONDUCT.md +84 -0
  7. data/Gemfile +5 -0
  8. data/LICENSE.txt +21 -0
  9. data/README.md +43 -0
  10. data/Rakefile +17 -0
  11. data/bin/console +15 -0
  12. data/bin/setup +8 -0
  13. data/fluent-plugin-perf-tools.gemspec +48 -0
  14. data/lib/fluent/plugin/in_perf_tools.rb +42 -0
  15. data/lib/fluent/plugin/perf_tools/cachestat.rb +65 -0
  16. data/lib/fluent/plugin/perf_tools/command.rb +30 -0
  17. data/lib/fluent/plugin/perf_tools/version.rb +9 -0
  18. data/lib/fluent/plugin/perf_tools.rb +11 -0
  19. data/perf-tools/LICENSE +339 -0
  20. data/perf-tools/README.md +205 -0
  21. data/perf-tools/bin/bitesize +1 -0
  22. data/perf-tools/bin/cachestat +1 -0
  23. data/perf-tools/bin/execsnoop +1 -0
  24. data/perf-tools/bin/funccount +1 -0
  25. data/perf-tools/bin/funcgraph +1 -0
  26. data/perf-tools/bin/funcslower +1 -0
  27. data/perf-tools/bin/functrace +1 -0
  28. data/perf-tools/bin/iolatency +1 -0
  29. data/perf-tools/bin/iosnoop +1 -0
  30. data/perf-tools/bin/killsnoop +1 -0
  31. data/perf-tools/bin/kprobe +1 -0
  32. data/perf-tools/bin/opensnoop +1 -0
  33. data/perf-tools/bin/perf-stat-hist +1 -0
  34. data/perf-tools/bin/reset-ftrace +1 -0
  35. data/perf-tools/bin/syscount +1 -0
  36. data/perf-tools/bin/tcpretrans +1 -0
  37. data/perf-tools/bin/tpoint +1 -0
  38. data/perf-tools/bin/uprobe +1 -0
  39. data/perf-tools/deprecated/README.md +1 -0
  40. data/perf-tools/deprecated/execsnoop-proc +150 -0
  41. data/perf-tools/deprecated/execsnoop-proc.8 +80 -0
  42. data/perf-tools/deprecated/execsnoop-proc_example.txt +46 -0
  43. data/perf-tools/disk/bitesize +175 -0
  44. data/perf-tools/examples/bitesize_example.txt +63 -0
  45. data/perf-tools/examples/cachestat_example.txt +58 -0
  46. data/perf-tools/examples/execsnoop_example.txt +153 -0
  47. data/perf-tools/examples/funccount_example.txt +126 -0
  48. data/perf-tools/examples/funcgraph_example.txt +2178 -0
  49. data/perf-tools/examples/funcslower_example.txt +110 -0
  50. data/perf-tools/examples/functrace_example.txt +341 -0
  51. data/perf-tools/examples/iolatency_example.txt +350 -0
  52. data/perf-tools/examples/iosnoop_example.txt +302 -0
  53. data/perf-tools/examples/killsnoop_example.txt +62 -0
  54. data/perf-tools/examples/kprobe_example.txt +379 -0
  55. data/perf-tools/examples/opensnoop_example.txt +47 -0
  56. data/perf-tools/examples/perf-stat-hist_example.txt +149 -0
  57. data/perf-tools/examples/reset-ftrace_example.txt +88 -0
  58. data/perf-tools/examples/syscount_example.txt +297 -0
  59. data/perf-tools/examples/tcpretrans_example.txt +93 -0
  60. data/perf-tools/examples/tpoint_example.txt +210 -0
  61. data/perf-tools/examples/uprobe_example.txt +321 -0
  62. data/perf-tools/execsnoop +292 -0
  63. data/perf-tools/fs/cachestat +167 -0
  64. data/perf-tools/images/perf-tools_2016.png +0 -0
  65. data/perf-tools/iolatency +296 -0
  66. data/perf-tools/iosnoop +296 -0
  67. data/perf-tools/kernel/funccount +146 -0
  68. data/perf-tools/kernel/funcgraph +259 -0
  69. data/perf-tools/kernel/funcslower +248 -0
  70. data/perf-tools/kernel/functrace +192 -0
  71. data/perf-tools/kernel/kprobe +270 -0
  72. data/perf-tools/killsnoop +263 -0
  73. data/perf-tools/man/man8/bitesize.8 +70 -0
  74. data/perf-tools/man/man8/cachestat.8 +111 -0
  75. data/perf-tools/man/man8/execsnoop.8 +104 -0
  76. data/perf-tools/man/man8/funccount.8 +76 -0
  77. data/perf-tools/man/man8/funcgraph.8 +166 -0
  78. data/perf-tools/man/man8/funcslower.8 +129 -0
  79. data/perf-tools/man/man8/functrace.8 +123 -0
  80. data/perf-tools/man/man8/iolatency.8 +116 -0
  81. data/perf-tools/man/man8/iosnoop.8 +169 -0
  82. data/perf-tools/man/man8/killsnoop.8 +100 -0
  83. data/perf-tools/man/man8/kprobe.8 +162 -0
  84. data/perf-tools/man/man8/opensnoop.8 +113 -0
  85. data/perf-tools/man/man8/perf-stat-hist.8 +111 -0
  86. data/perf-tools/man/man8/reset-ftrace.8 +49 -0
  87. data/perf-tools/man/man8/syscount.8 +96 -0
  88. data/perf-tools/man/man8/tcpretrans.8 +93 -0
  89. data/perf-tools/man/man8/tpoint.8 +140 -0
  90. data/perf-tools/man/man8/uprobe.8 +168 -0
  91. data/perf-tools/misc/perf-stat-hist +223 -0
  92. data/perf-tools/net/tcpretrans +311 -0
  93. data/perf-tools/opensnoop +280 -0
  94. data/perf-tools/syscount +192 -0
  95. data/perf-tools/system/tpoint +232 -0
  96. data/perf-tools/tools/reset-ftrace +123 -0
  97. data/perf-tools/user/uprobe +390 -0
  98. metadata +349 -0
@@ -0,0 +1,167 @@
1
+ #!/bin/bash
2
+ #
3
+ # cachestat - show Linux page cache hit/miss statistics.
4
+ # Uses Linux ftrace.
5
+ #
6
+ # This is a proof of concept using Linux ftrace capabilities on older kernels,
7
+ # and works by using function profiling for in-kernel counters. Specifically,
8
+ # four kernel functions are traced:
9
+ #
10
+ # mark_page_accessed() for measuring cache accesses
11
+ # mark_buffer_dirty() for measuring cache writes
12
+ # add_to_page_cache_lru() for measuring page additions
13
+ # account_page_dirtied() for measuring page dirties
14
+ #
15
+ # It is possible that these functions have been renamed (or are different
16
+ # logically) for your kernel version, and this script will not work as-is.
17
+ # This script was written on Linux 3.13. This script is a sandcastle: the
18
+ # kernel may wash some away, and you'll need to rebuild.
19
+ #
20
+ # USAGE: cachestat [-Dht] [interval]
21
+ # eg,
22
+ # cachestat 5 # show stats every 5 seconds
23
+ #
24
+ # Run "cachestat -h" for full usage.
25
+ #
26
+ # WARNING: This uses dynamic tracing of kernel functions, and could cause
27
+ # kernel panics or freezes. Test, and know what you are doing, before use.
28
+ # It also traces cache activity, which can be frequent, and cost some overhead.
29
+ # The statistics should be treated as best-effort: there may be some error
30
+ # margin depending on unusual workload types.
31
+ #
32
+ # REQUIREMENTS: CONFIG_FUNCTION_PROFILER, awk.
33
+ #
34
+ # From perf-tools: https://github.com/brendangregg/perf-tools
35
+ #
36
+ # COPYRIGHT: Copyright (c) 2014 Brendan Gregg.
37
+ #
38
+ # This program is free software; you can redistribute it and/or
39
+ # modify it under the terms of the GNU General Public License
40
+ # as published by the Free Software Foundation; either version 2
41
+ # of the License, or (at your option) any later version.
42
+ #
43
+ # This program is distributed in the hope that it will be useful,
44
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
45
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
46
+ # GNU General Public License for more details.
47
+ #
48
+ # You should have received a copy of the GNU General Public License
49
+ # along with this program; if not, write to the Free Software Foundation,
50
+ # Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
51
+ #
52
+ # (http://www.gnu.org/copyleft/gpl.html)
53
+ #
54
+ # 28-Dec-2014 Brendan Gregg Created this.
55
+
56
+ ### default variables
57
+ tracing=/sys/kernel/debug/tracing
58
+ interval=1; opt_timestamp=0; opt_debug=0
59
+ trap 'quit=1' INT QUIT TERM PIPE HUP # sends execution to end tracing section
60
+
61
+ function usage {
62
+ cat <<-END >&2
63
+ USAGE: cachestat [-Dht] [interval]
64
+ -D # print debug counters
65
+ -h # this usage message
66
+ -t # include timestamp
67
+ interval # output interval in secs (default 1)
68
+ eg,
69
+ cachestat # show stats every second
70
+ cachestat 5 # show stats every 5 seconds
71
+
72
+ See the man page and example file for more info.
73
+ END
74
+ exit
75
+ }
76
+
77
+ function warn {
78
+ if ! eval "$@"; then
79
+ echo >&2 "WARNING: command failed \"$@\""
80
+ fi
81
+ }
82
+
83
+ function die {
84
+ echo >&2 "$@"
85
+ exit 1
86
+ }
87
+
88
+ ### process options
89
+ while getopts Dht opt
90
+ do
91
+ case $opt in
92
+ D) opt_debug=1 ;;
93
+ t) opt_timestamp=1 ;;
94
+ h|?) usage ;;
95
+ esac
96
+ done
97
+ shift $(( $OPTIND - 1 ))
98
+
99
+ ### option logic
100
+ if (( $# )); then
101
+ interval=$1
102
+ fi
103
+ echo "Counting cache functions... Output every $interval seconds."
104
+
105
+ ### check permissions
106
+ cd $tracing || die "ERROR: accessing tracing. Root user? Kernel has FTRACE?
107
+ debugfs mounted? (mount -t debugfs debugfs /sys/kernel/debug)"
108
+
109
+ ### enable tracing
110
+ sysctl -q kernel.ftrace_enabled=1 # doesn't set exit status
111
+ printf "mark_page_accessed\nmark_buffer_dirty\nadd_to_page_cache_lru\naccount_page_dirtied\n" > set_ftrace_filter || \
112
+ die "ERROR: tracing these four kernel functions: mark_page_accessed,"\
113
+ "mark_buffer_dirty, add_to_page_cache_lru and account_page_dirtied (unknown kernel version?). Exiting."
114
+ warn "echo nop > current_tracer"
115
+ if ! echo 1 > function_profile_enabled; then
116
+ echo > set_ftrace_filter
117
+ die "ERROR: enabling function profiling. Have CONFIG_FUNCTION_PROFILER? Exiting."
118
+ fi
119
+
120
+ (( opt_timestamp )) && printf "%-8s " TIME
121
+ printf "%8s %8s %8s %8s %12s %10s" HITS MISSES DIRTIES RATIO "BUFFERS_MB" "CACHE_MB"
122
+ (( opt_debug )) && printf " DEBUG"
123
+ echo
124
+
125
+ ### summarize
126
+ quit=0; secs=0
127
+ while (( !quit && (!opt_duration || secs < duration) )); do
128
+ (( secs += interval ))
129
+ echo 0 > function_profile_enabled
130
+ echo 1 > function_profile_enabled
131
+ sleep $interval
132
+
133
+ (( opt_timestamp )) && printf "%(%H:%M:%S)T " -1
134
+
135
+ # cat both meminfo and trace stats, and let awk pick them apart
136
+ cat /proc/meminfo trace_stat/function* | awk -v debug=$opt_debug '
137
+ # match meminfo stats:
138
+ $1 == "Buffers:" && $3 == "kB" { buffers_mb = $2 / 1024 }
139
+ $1 == "Cached:" && $3 == "kB" { cached_mb = $2 / 1024 }
140
+ # identify and save trace counts:
141
+ $2 ~ /[0-9]/ && $3 != "kB" { a[$1] += $2 }
142
+ END {
143
+ mpa = a["mark_page_accessed"]
144
+ mbd = a["mark_buffer_dirty"]
145
+ apcl = a["add_to_page_cache_lru"]
146
+ apd = a["account_page_dirtied"]
147
+
148
+ total = mpa - mbd
149
+ misses = apcl - apd
150
+ if (misses < 0)
151
+ misses = 0
152
+ hits = total - misses
153
+
154
+ ratio = 100 * hits / total
155
+ printf "%8d %8d %8d %7.1f%% %12.0f %10.0f", hits, misses, mbd,
156
+ ratio, buffers_mb, cached_mb
157
+ if (debug)
158
+ printf " (%d %d %d %d)", mpa, mbd, apcl, apd
159
+ printf "\n"
160
+ }'
161
+ done
162
+
163
+ ### end tracing
164
+ echo 2>/dev/null
165
+ echo "Ending tracing..." 2>/dev/null
166
+ warn "echo 0 > function_profile_enabled"
167
+ warn "echo > set_ftrace_filter"
@@ -0,0 +1,296 @@
1
+ #!/bin/bash
2
+ #
3
+ # iolatency - summarize block device I/O latency as a histogram.
4
+ # Written using Linux ftrace.
5
+ #
6
+ # This shows the distribution of latency, allowing modes and latency outliers
7
+ # to be identified and studied.
8
+ #
9
+ # USAGE: ./iolatency [-hQT] [-d device] [-i iotype] [interval [count]]
10
+ #
11
+ # REQUIREMENTS: FTRACE CONFIG and block:block_rq_* tracepoints, which you may
12
+ # already have on recent kernels.
13
+ #
14
+ # OVERHEAD: block device I/O issue and completion events are traced and buffered
15
+ # in-kernel, then processed and summarized in user space. There may be
16
+ # measurable overhead with this approach, relative to the block device IOPS.
17
+ #
18
+ # This was written as a proof of concept for ftrace.
19
+ #
20
+ # From perf-tools: https://github.com/brendangregg/perf-tools
21
+ #
22
+ # COPYRIGHT: Copyright (c) 2014 Brendan Gregg.
23
+ #
24
+ # This program is free software; you can redistribute it and/or
25
+ # modify it under the terms of the GNU General Public License
26
+ # as published by the Free Software Foundation; either version 2
27
+ # of the License, or (at your option) any later version.
28
+ #
29
+ # This program is distributed in the hope that it will be useful,
30
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
31
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
32
+ # GNU General Public License for more details.
33
+ #
34
+ # You should have received a copy of the GNU General Public License
35
+ # along with this program; if not, write to the Free Software Foundation,
36
+ # Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
37
+ #
38
+ # (http://www.gnu.org/copyleft/gpl.html)
39
+ #
40
+ # 20-Jul-2014 Brendan Gregg Created this.
41
+
42
+ ### default variables
43
+ tracing=/sys/kernel/debug/tracing
44
+ flock=/var/tmp/.ftrace-lock
45
+ bufsize_kb=4096
46
+ opt_device=0; device=; opt_iotype=0; iotype=; opt_timestamp=0
47
+ opt_interval=0; interval=1; opt_count=0; count=0; opt_queue=0
48
+ trap ':' INT QUIT TERM PIPE HUP # sends execution to end tracing section
49
+
50
+ function usage {
51
+ cat <<-END >&2
52
+ USAGE: iolatency [-hQT] [-d device] [-i iotype] [interval [count]]
53
+ -d device # device string (eg, "202,1)
54
+ -i iotype # match type (eg, '*R*' for all reads)
55
+ -Q # use queue insert as start time
56
+ -T # timestamp on output
57
+ -h # this usage message
58
+ interval # summary interval, seconds (default 1)
59
+ count # number of summaries
60
+ eg,
61
+ iolatency # summarize latency every second
62
+ iolatency -Q # include block I/O queue time
63
+ iolatency 5 2 # 2 x 5 second summaries
64
+ iolatency -i '*R*' # trace reads
65
+ iolatency -d 202,1 # trace device 202,1 only
66
+
67
+ See the man page and example file for more info.
68
+ END
69
+ exit
70
+ }
71
+
72
+ function warn {
73
+ if ! eval "$@"; then
74
+ echo >&2 "WARNING: command failed \"$@\""
75
+ fi
76
+ }
77
+
78
+ function end {
79
+ # disable tracing
80
+ echo 2>/dev/null
81
+ echo "Ending tracing..." 2>/dev/null
82
+ cd $tracing
83
+ warn "echo 0 > events/block/$b_start/enable"
84
+ warn "echo 0 > events/block/block_rq_complete/enable"
85
+ if (( opt_device || opt_iotype )); then
86
+ warn "echo 0 > events/block/$b_start/filter"
87
+ warn "echo 0 > events/block/block_rq_complete/filter"
88
+ fi
89
+ warn "echo > trace"
90
+ (( wroteflock )) && warn "rm $flock"
91
+ }
92
+
93
+ function die {
94
+ echo >&2 "$@"
95
+ exit 1
96
+ }
97
+
98
+ function edie {
99
+ # die with a quiet end()
100
+ echo >&2 "$@"
101
+ exec >/dev/null 2>&1
102
+ end
103
+ exit 1
104
+ }
105
+
106
+ ### process options
107
+ while getopts d:hi:QT opt
108
+ do
109
+ case $opt in
110
+ d) opt_device=1; device=$OPTARG ;;
111
+ i) opt_iotype=1; iotype=$OPTARG ;;
112
+ Q) opt_queue=1 ;;
113
+ T) opt_timestamp=1 ;;
114
+ h|?) usage ;;
115
+ esac
116
+ done
117
+ shift $(( $OPTIND - 1 ))
118
+ if (( $# )); then
119
+ opt_interval=1
120
+ interval=$1
121
+ shift
122
+ fi
123
+ if (( $# )); then
124
+ opt_count=1
125
+ count=$1
126
+ fi
127
+ if (( opt_device )); then
128
+ major=${device%,*}
129
+ minor=${device#*,}
130
+ dev=$(( (major << 20) + minor ))
131
+ fi
132
+ if (( opt_queue )); then
133
+ b_start=block_rq_insert
134
+ else
135
+ b_start=block_rq_issue
136
+ fi
137
+
138
+ ### select awk
139
+ [[ -x /usr/bin/mawk ]] && awk='mawk -W interactive' || awk=awk
140
+
141
+ ### check permissions
142
+ cd $tracing || die "ERROR: accessing tracing. Root user? Kernel has FTRACE?
143
+ debugfs mounted? (mount -t debugfs debugfs /sys/kernel/debug)"
144
+
145
+ ### ftrace lock
146
+ [[ -e $flock ]] && die "ERROR: ftrace may be in use by PID $(cat $flock) $flock"
147
+ echo $$ > $flock || die "ERROR: unable to write $flock."
148
+ wroteflock=1
149
+
150
+ ### setup and begin tracing
151
+ warn "echo nop > current_tracer"
152
+ warn "echo $bufsize_kb > buffer_size_kb"
153
+ filter=
154
+ if (( opt_iotype )); then
155
+ filter="rwbs ~ \"$iotype\""
156
+ fi
157
+ if (( opt_device )); then
158
+ [[ "$filter" != "" ]] && filter="$filter && "
159
+ filter="${filter}dev == $dev"
160
+ fi
161
+ if (( opt_iotype || opt_device )); then
162
+ if ! echo "$filter" > events/block/$b_start/filter || \
163
+ ! echo "$filter" > events/block/block_rq_complete/filter
164
+ then
165
+ edie "ERROR: setting -d or -t filter. Exiting."
166
+ fi
167
+ fi
168
+ if ! echo 1 > events/block/$b_start/enable || \
169
+ ! echo 1 > events/block/block_rq_complete/enable; then
170
+ edie "ERROR: enabling block I/O tracepoints. Exiting."
171
+ fi
172
+ etext=
173
+ (( !opt_count )) && etext=" Ctrl-C to end."
174
+ echo "Tracing block I/O. Output every $interval seconds.$etext"
175
+
176
+ #
177
+ # Determine output format. It may be one of the following (newest first):
178
+ # TASK-PID CPU# |||| TIMESTAMP FUNCTION
179
+ # TASK-PID CPU# TIMESTAMP FUNCTION
180
+ # To differentiate between them, the number of header fields is counted,
181
+ # and an offset set, to skip the extra column when needed.
182
+ #
183
+ offset=$($awk 'BEGIN { o = 0; }
184
+ $1 == "#" && $2 ~ /TASK/ && NF == 6 { o = 1; }
185
+ $2 ~ /TASK/ { print o; exit }' trace)
186
+
187
+ ### print trace buffer
188
+ warn "echo > trace"
189
+ i=0
190
+ while (( !opt_count || (i < count) )); do
191
+ (( i++ ))
192
+ sleep $interval
193
+
194
+ # snapshots were added in 3.10
195
+ if [[ -x snapshot ]]; then
196
+ echo 1 > snapshot
197
+ echo > trace
198
+ cat snapshot
199
+ else
200
+ cat trace
201
+ echo > trace
202
+ fi
203
+
204
+ (( opt_timestamp )) && printf "time %(%H:%M:%S)T:\n" -1
205
+ echo "tick"
206
+ done | \
207
+ $awk -v o=$offset -v opt_timestamp=$opt_timestamp -v b_start=$b_start '
208
+ function star(sval, smax, swidth) {
209
+ stars = ""
210
+ if (smax == 0) return ""
211
+ for (si = 0; si < (swidth * sval / smax); si++) {
212
+ stars = stars "#"
213
+ }
214
+ return stars
215
+ }
216
+
217
+ BEGIN { max_i = 0 }
218
+
219
+ # common fields
220
+ $1 != "#" {
221
+ time = $(3+o); sub(":", "", time)
222
+ dev = $(5+o)
223
+ }
224
+
225
+ # block I/O request
226
+ $1 != "#" && $0 ~ b_start {
227
+ #
228
+ # example: (fields1..4+o) 202,1 W 0 () 12862264 + 8 [tar]
229
+ # The cmd field "()" might contain multiple words (hex),
230
+ # hence stepping from the right (NF-3).
231
+ #
232
+ loc = $(NF-3)
233
+ starts[dev, loc] = time
234
+ next
235
+ }
236
+
237
+ # block I/O completion
238
+ $1 != "#" && $0 ~ /rq_complete/ {
239
+ #
240
+ # example: (fields1..4+o) 202,1 W () 12862256 + 8 [0]
241
+ #
242
+ dir = $(6+o)
243
+ loc = $(NF-3)
244
+
245
+ if (starts[dev, loc] > 0) {
246
+ latency_ms = 1000 * (time - starts[dev, loc])
247
+ i = 0
248
+ for (ms = 1; latency_ms > ms; ms *= 2) { i++ }
249
+ hist[i]++
250
+ if (i > max_i)
251
+ max_i = i
252
+ delete starts[dev, loc]
253
+ }
254
+ next
255
+ }
256
+
257
+ # timestamp
258
+ $1 == "time" {
259
+ lasttime = $2
260
+ }
261
+
262
+ # print summary
263
+ $1 == "tick" {
264
+ print ""
265
+ if (opt_timestamp)
266
+ print lasttime
267
+
268
+ # find max value
269
+ max_v = 0
270
+ for (i = 0; i <= max_i; i++) {
271
+ if (hist[i] > max_v)
272
+ max_v = hist[i]
273
+ }
274
+
275
+ # print histogram
276
+ printf "%8s .. %-8s: %-8s |%-38s|\n", ">=(ms)", "<(ms)",
277
+ "I/O", "Distribution"
278
+ ms = 1
279
+ from = 0
280
+ for (i = 0; i <= max_i; i++) {
281
+ printf "%8d -> %-8d: %-8d |%-38s|\n", from, ms,
282
+ hist[i], star(hist[i], max_v, 38)
283
+ from = ms
284
+ ms *= 2
285
+ }
286
+ fflush()
287
+ delete hist
288
+ delete starts # invalid if events missed between snapshots
289
+ max_i = 0
290
+ }
291
+
292
+ $0 ~ /LOST.*EVENTS/ { print "WARNING: " $0 > "/dev/stderr" }
293
+ '
294
+
295
+ ### end tracing
296
+ end