stackprof 0.2.2 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,3266 @@
1
+ #!/usr/bin/env python
2
+ #
3
+ # Copyright 2008-2009 Jose Fonseca
4
+ #
5
+ # This program is free software: you can redistribute it and/or modify it
6
+ # under the terms of the GNU Lesser General Public License as published
7
+ # by the Free Software Foundation, either version 3 of the License, or
8
+ # (at your option) any later version.
9
+ #
10
+ # This program is distributed in the hope that it will be useful,
11
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ # GNU Lesser General Public License for more details.
14
+ #
15
+ # You should have received a copy of the GNU Lesser General Public License
16
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
+ #
18
+
19
+ """Generate a dot graph from the output of several profilers."""
20
+
21
+ __author__ = "Jose Fonseca et al"
22
+
23
+
24
+ import sys
25
+ import math
26
+ import os.path
27
+ import re
28
+ import textwrap
29
+ import optparse
30
+ import xml.parsers.expat
31
+ import collections
32
+ import locale
33
+
34
+
35
+ # Python 2.x/3.x compatibility
36
+ if sys.version_info[0] >= 3:
37
+ PYTHON_3 = True
38
+ def compat_iteritems(x): return x.items() # No iteritems() in Python 3
39
+ def compat_itervalues(x): return x.values() # No itervalues() in Python 3
40
+ def compat_keys(x): return list(x.keys()) # keys() is a generator in Python 3
41
+ basestring = str # No class basestring in Python 3
42
+ unichr = chr # No unichr in Python 3
43
+ xrange = range # No xrange in Python 3
44
+ else:
45
+ PYTHON_3 = False
46
+ def compat_iteritems(x): return x.iteritems()
47
+ def compat_itervalues(x): return x.itervalues()
48
+ def compat_keys(x): return x.keys()
49
+
50
+
51
+ try:
52
+ # Debugging helper module
53
+ import debug
54
+ except ImportError:
55
+ pass
56
+
57
+
58
+ MULTIPLICATION_SIGN = unichr(0xd7)
59
+
60
+
61
+ def times(x):
62
+ return "%u%s" % (x, MULTIPLICATION_SIGN)
63
+
64
+ def percentage(p):
65
+ return "%.02f%%" % (p*100.0,)
66
+
67
+ def add(a, b):
68
+ return a + b
69
+
70
+ def equal(a, b):
71
+ if a == b:
72
+ return a
73
+ else:
74
+ return None
75
+
76
+ def fail(a, b):
77
+ assert False
78
+
79
+
80
+ tol = 2 ** -23
81
+
82
+ def ratio(numerator, denominator):
83
+ try:
84
+ ratio = float(numerator)/float(denominator)
85
+ except ZeroDivisionError:
86
+ # 0/0 is undefined, but 1.0 yields more useful results
87
+ return 1.0
88
+ if ratio < 0.0:
89
+ if ratio < -tol:
90
+ sys.stderr.write('warning: negative ratio (%s/%s)\n' % (numerator, denominator))
91
+ return 0.0
92
+ if ratio > 1.0:
93
+ if ratio > 1.0 + tol:
94
+ sys.stderr.write('warning: ratio greater than one (%s/%s)\n' % (numerator, denominator))
95
+ return 1.0
96
+ return ratio
97
+
98
+
99
+ class UndefinedEvent(Exception):
100
+ """Raised when attempting to get an event which is undefined."""
101
+
102
+ def __init__(self, event):
103
+ Exception.__init__(self)
104
+ self.event = event
105
+
106
+ def __str__(self):
107
+ return 'unspecified event %s' % self.event.name
108
+
109
+
110
+ class Event(object):
111
+ """Describe a kind of event, and its basic operations."""
112
+
113
+ def __init__(self, name, null, aggregator, formatter = str):
114
+ self.name = name
115
+ self._null = null
116
+ self._aggregator = aggregator
117
+ self._formatter = formatter
118
+
119
+ def __eq__(self, other):
120
+ return self is other
121
+
122
+ def __hash__(self):
123
+ return id(self)
124
+
125
+ def null(self):
126
+ return self._null
127
+
128
+ def aggregate(self, val1, val2):
129
+ """Aggregate two event values."""
130
+ assert val1 is not None
131
+ assert val2 is not None
132
+ return self._aggregator(val1, val2)
133
+
134
+ def format(self, val):
135
+ """Format an event value."""
136
+ assert val is not None
137
+ return self._formatter(val)
138
+
139
+
140
+ CALLS = Event("Calls", 0, add, times)
141
+ SAMPLES = Event("Samples", 0, add, times)
142
+ SAMPLES2 = Event("Samples", 0, add, times)
143
+
144
+ # Count of samples where a given function was either executing or on the stack.
145
+ # This is used to calculate the total time ratio according to the
146
+ # straightforward method described in Mike Dunlavey's answer to
147
+ # stackoverflow.com/questions/1777556/alternatives-to-gprof, item 4 (the myth
148
+ # "that recursion is a tricky confusing issue"), last edited 2012-08-30: it's
149
+ # just the ratio of TOTAL_SAMPLES over the number of samples in the profile.
150
+ #
151
+ # Used only when totalMethod == callstacks
152
+ TOTAL_SAMPLES = Event("Samples", 0, add, times)
153
+
154
+ TIME = Event("Time", 0.0, add, lambda x: '(' + str(x) + ')')
155
+ TIME_RATIO = Event("Time ratio", 0.0, add, lambda x: '(' + percentage(x) + ')')
156
+ TOTAL_TIME = Event("Total time", 0.0, fail)
157
+ TOTAL_TIME_RATIO = Event("Total time ratio", 0.0, fail, percentage)
158
+
159
+ totalMethod = 'callratios'
160
+
161
+
162
+ class Object(object):
163
+ """Base class for all objects in profile which can store events."""
164
+
165
+ def __init__(self, events=None):
166
+ if events is None:
167
+ self.events = {}
168
+ else:
169
+ self.events = events
170
+
171
+ def __hash__(self):
172
+ return id(self)
173
+
174
+ def __eq__(self, other):
175
+ return self is other
176
+
177
+ def __contains__(self, event):
178
+ return event in self.events
179
+
180
+ def __getitem__(self, event):
181
+ try:
182
+ return self.events[event]
183
+ except KeyError:
184
+ raise UndefinedEvent(event)
185
+
186
+ def __setitem__(self, event, value):
187
+ if value is None:
188
+ if event in self.events:
189
+ del self.events[event]
190
+ else:
191
+ self.events[event] = value
192
+
193
+
194
+ class Call(Object):
195
+ """A call between functions.
196
+
197
+ There should be at most one call object for every pair of functions.
198
+ """
199
+
200
+ def __init__(self, callee_id):
201
+ Object.__init__(self)
202
+ self.callee_id = callee_id
203
+ self.ratio = None
204
+ self.weight = None
205
+
206
+
207
+ class Function(Object):
208
+ """A function."""
209
+
210
+ def __init__(self, id, name):
211
+ Object.__init__(self)
212
+ self.id = id
213
+ self.name = name
214
+ self.module = None
215
+ self.process = None
216
+ self.calls = {}
217
+ self.called = None
218
+ self.weight = None
219
+ self.cycle = None
220
+
221
+ def add_call(self, call):
222
+ if call.callee_id in self.calls:
223
+ sys.stderr.write('warning: overwriting call from function %s to %s\n' % (str(self.id), str(call.callee_id)))
224
+ self.calls[call.callee_id] = call
225
+
226
+ def get_call(self, callee_id):
227
+ if not callee_id in self.calls:
228
+ call = Call(callee_id)
229
+ call[SAMPLES] = 0
230
+ call[SAMPLES2] = 0
231
+ call[CALLS] = 0
232
+ self.calls[callee_id] = call
233
+ return self.calls[callee_id]
234
+
235
+ _parenthesis_re = re.compile(r'\([^()]*\)')
236
+ _angles_re = re.compile(r'<[^<>]*>')
237
+ _const_re = re.compile(r'\s+const$')
238
+
239
+ def stripped_name(self):
240
+ """Remove extraneous information from C++ demangled function names."""
241
+
242
+ name = self.name
243
+
244
+ # Strip function parameters from name by recursively removing paired parenthesis
245
+ while True:
246
+ name, n = self._parenthesis_re.subn('', name)
247
+ if not n:
248
+ break
249
+
250
+ # Strip const qualifier
251
+ name = self._const_re.sub('', name)
252
+
253
+ # Strip template parameters from name by recursively removing paired angles
254
+ while True:
255
+ name, n = self._angles_re.subn('', name)
256
+ if not n:
257
+ break
258
+
259
+ return name
260
+
261
+ # TODO: write utility functions
262
+
263
+ def __repr__(self):
264
+ return self.name
265
+
266
+
267
+ class Cycle(Object):
268
+ """A cycle made from recursive function calls."""
269
+
270
+ def __init__(self):
271
+ Object.__init__(self)
272
+ # XXX: Do cycles need an id?
273
+ self.functions = set()
274
+
275
+ def add_function(self, function):
276
+ assert function not in self.functions
277
+ self.functions.add(function)
278
+ # XXX: Aggregate events?
279
+ if function.cycle is not None:
280
+ for other in function.cycle.functions:
281
+ if function not in self.functions:
282
+ self.add_function(other)
283
+ function.cycle = self
284
+
285
+
286
+ class Profile(Object):
287
+ """The whole profile."""
288
+
289
+ def __init__(self):
290
+ Object.__init__(self)
291
+ self.functions = {}
292
+ self.cycles = []
293
+
294
+ def add_function(self, function):
295
+ if function.id in self.functions:
296
+ sys.stderr.write('warning: overwriting function %s (id %s)\n' % (function.name, str(function.id)))
297
+ self.functions[function.id] = function
298
+
299
+ def add_cycle(self, cycle):
300
+ self.cycles.append(cycle)
301
+
302
+ def validate(self):
303
+ """Validate the edges."""
304
+
305
+ for function in compat_itervalues(self.functions):
306
+ for callee_id in compat_keys(function.calls):
307
+ assert function.calls[callee_id].callee_id == callee_id
308
+ if callee_id not in self.functions:
309
+ sys.stderr.write('warning: call to undefined function %s from function %s\n' % (str(callee_id), function.name))
310
+ del function.calls[callee_id]
311
+
312
+ def find_cycles(self):
313
+ """Find cycles using Tarjan's strongly connected components algorithm."""
314
+
315
+ # Apply the Tarjan's algorithm successively until all functions are visited
316
+ visited = set()
317
+ for function in compat_itervalues(self.functions):
318
+ if function not in visited:
319
+ self._tarjan(function, 0, [], {}, {}, visited)
320
+ cycles = []
321
+ for function in compat_itervalues(self.functions):
322
+ if function.cycle is not None and function.cycle not in cycles:
323
+ cycles.append(function.cycle)
324
+ self.cycles = cycles
325
+ if 0:
326
+ for cycle in cycles:
327
+ sys.stderr.write("Cycle:\n")
328
+ for member in cycle.functions:
329
+ sys.stderr.write("\tFunction %s\n" % member.name)
330
+
331
+ def prune_root(self, root):
332
+ visited = set()
333
+ frontier = set([root])
334
+ while len(frontier) > 0:
335
+ node = frontier.pop()
336
+ visited.add(node)
337
+ f = self.functions[node]
338
+ newNodes = f.calls.keys()
339
+ frontier = frontier.union(set(newNodes) - visited)
340
+ subtreeFunctions = {}
341
+ for n in visited:
342
+ subtreeFunctions[n] = self.functions[n]
343
+ self.functions = subtreeFunctions
344
+
345
+ def prune_leaf(self, leaf):
346
+ edgesUp = collections.defaultdict(set)
347
+ for f in self.functions.keys():
348
+ for n in self.functions[f].calls.keys():
349
+ edgesUp[n].add(f)
350
+ # build the tree up
351
+ visited = set()
352
+ frontier = set([leaf])
353
+ while len(frontier) > 0:
354
+ node = frontier.pop()
355
+ visited.add(node)
356
+ frontier = frontier.union(edgesUp[node] - visited)
357
+ downTree = set(self.functions.keys())
358
+ upTree = visited
359
+ path = downTree.intersection(upTree)
360
+ pathFunctions = {}
361
+ for n in path:
362
+ f = self.functions[n]
363
+ newCalls = {}
364
+ for c in f.calls.keys():
365
+ if c in path:
366
+ newCalls[c] = f.calls[c]
367
+ f.calls = newCalls
368
+ pathFunctions[n] = f
369
+ self.functions = pathFunctions
370
+
371
+
372
+ def getFunctionId(self, funcName):
373
+ for f in self.functions:
374
+ if self.functions[f].name == funcName:
375
+ return f
376
+ return False
377
+
378
+ def _tarjan(self, function, order, stack, orders, lowlinks, visited):
379
+ """Tarjan's strongly connected components algorithm.
380
+
381
+ See also:
382
+ - http://en.wikipedia.org/wiki/Tarjan's_strongly_connected_components_algorithm
383
+ """
384
+
385
+ visited.add(function)
386
+ orders[function] = order
387
+ lowlinks[function] = order
388
+ order += 1
389
+ pos = len(stack)
390
+ stack.append(function)
391
+ for call in compat_itervalues(function.calls):
392
+ callee = self.functions[call.callee_id]
393
+ # TODO: use a set to optimize lookup
394
+ if callee not in orders:
395
+ order = self._tarjan(callee, order, stack, orders, lowlinks, visited)
396
+ lowlinks[function] = min(lowlinks[function], lowlinks[callee])
397
+ elif callee in stack:
398
+ lowlinks[function] = min(lowlinks[function], orders[callee])
399
+ if lowlinks[function] == orders[function]:
400
+ # Strongly connected component found
401
+ members = stack[pos:]
402
+ del stack[pos:]
403
+ if len(members) > 1:
404
+ cycle = Cycle()
405
+ for member in members:
406
+ cycle.add_function(member)
407
+ return order
408
+
409
+ def call_ratios(self, event):
410
+ # Aggregate for incoming calls
411
+ cycle_totals = {}
412
+ for cycle in self.cycles:
413
+ cycle_totals[cycle] = 0.0
414
+ function_totals = {}
415
+ for function in compat_itervalues(self.functions):
416
+ function_totals[function] = 0.0
417
+
418
+ # Pass 1: function_total gets the sum of call[event] for all
419
+ # incoming arrows. Same for cycle_total for all arrows
420
+ # that are coming into the *cycle* but are not part of it.
421
+ for function in compat_itervalues(self.functions):
422
+ for call in compat_itervalues(function.calls):
423
+ if call.callee_id != function.id:
424
+ callee = self.functions[call.callee_id]
425
+ if event in call.events:
426
+ function_totals[callee] += call[event]
427
+ if callee.cycle is not None and callee.cycle is not function.cycle:
428
+ cycle_totals[callee.cycle] += call[event]
429
+ else:
430
+ sys.stderr.write("call_ratios: No data for " + function.name + " call to " + callee.name + "\n")
431
+
432
+ # Pass 2: Compute the ratios. Each call[event] is scaled by the
433
+ # function_total of the callee. Calls into cycles use the
434
+ # cycle_total, but not calls within cycles.
435
+ for function in compat_itervalues(self.functions):
436
+ for call in compat_itervalues(function.calls):
437
+ assert call.ratio is None
438
+ if call.callee_id != function.id:
439
+ callee = self.functions[call.callee_id]
440
+ if event in call.events:
441
+ if callee.cycle is not None and callee.cycle is not function.cycle:
442
+ total = cycle_totals[callee.cycle]
443
+ else:
444
+ total = function_totals[callee]
445
+ call.ratio = ratio(call[event], total)
446
+ else:
447
+ # Warnings here would only repeat those issued above.
448
+ call.ratio = 0.0
449
+
450
+ def integrate(self, outevent, inevent):
451
+ """Propagate function time ratio along the function calls.
452
+
453
+ Must be called after finding the cycles.
454
+
455
+ See also:
456
+ - http://citeseer.ist.psu.edu/graham82gprof.html
457
+ """
458
+
459
+ # Sanity checking
460
+ assert outevent not in self
461
+ for function in compat_itervalues(self.functions):
462
+ assert outevent not in function
463
+ assert inevent in function
464
+ for call in compat_itervalues(function.calls):
465
+ assert outevent not in call
466
+ if call.callee_id != function.id:
467
+ assert call.ratio is not None
468
+
469
+ # Aggregate the input for each cycle
470
+ for cycle in self.cycles:
471
+ total = inevent.null()
472
+ for function in compat_itervalues(self.functions):
473
+ total = inevent.aggregate(total, function[inevent])
474
+ self[inevent] = total
475
+
476
+ # Integrate along the edges
477
+ total = inevent.null()
478
+ for function in compat_itervalues(self.functions):
479
+ total = inevent.aggregate(total, function[inevent])
480
+ self._integrate_function(function, outevent, inevent)
481
+ self[outevent] = total
482
+
483
+ def _integrate_function(self, function, outevent, inevent):
484
+ if function.cycle is not None:
485
+ return self._integrate_cycle(function.cycle, outevent, inevent)
486
+ else:
487
+ if outevent not in function:
488
+ total = function[inevent]
489
+ for call in compat_itervalues(function.calls):
490
+ if call.callee_id != function.id:
491
+ total += self._integrate_call(call, outevent, inevent)
492
+ function[outevent] = total
493
+ return function[outevent]
494
+
495
+ def _integrate_call(self, call, outevent, inevent):
496
+ assert outevent not in call
497
+ assert call.ratio is not None
498
+ callee = self.functions[call.callee_id]
499
+ subtotal = call.ratio *self._integrate_function(callee, outevent, inevent)
500
+ call[outevent] = subtotal
501
+ return subtotal
502
+
503
+ def _integrate_cycle(self, cycle, outevent, inevent):
504
+ if outevent not in cycle:
505
+
506
+ # Compute the outevent for the whole cycle
507
+ total = inevent.null()
508
+ for member in cycle.functions:
509
+ subtotal = member[inevent]
510
+ for call in compat_itervalues(member.calls):
511
+ callee = self.functions[call.callee_id]
512
+ if callee.cycle is not cycle:
513
+ subtotal += self._integrate_call(call, outevent, inevent)
514
+ total += subtotal
515
+ cycle[outevent] = total
516
+
517
+ # Compute the time propagated to callers of this cycle
518
+ callees = {}
519
+ for function in compat_itervalues(self.functions):
520
+ if function.cycle is not cycle:
521
+ for call in compat_itervalues(function.calls):
522
+ callee = self.functions[call.callee_id]
523
+ if callee.cycle is cycle:
524
+ try:
525
+ callees[callee] += call.ratio
526
+ except KeyError:
527
+ callees[callee] = call.ratio
528
+
529
+ for member in cycle.functions:
530
+ member[outevent] = outevent.null()
531
+
532
+ for callee, call_ratio in compat_iteritems(callees):
533
+ ranks = {}
534
+ call_ratios = {}
535
+ partials = {}
536
+ self._rank_cycle_function(cycle, callee, 0, ranks)
537
+ self._call_ratios_cycle(cycle, callee, ranks, call_ratios, set())
538
+ partial = self._integrate_cycle_function(cycle, callee, call_ratio, partials, ranks, call_ratios, outevent, inevent)
539
+ assert partial == max(partials.values())
540
+ assert not total or abs(1.0 - partial/(call_ratio*total)) <= 0.001
541
+
542
+ return cycle[outevent]
543
+
544
+ def _rank_cycle_function(self, cycle, function, rank, ranks):
545
+ if function not in ranks or ranks[function] > rank:
546
+ ranks[function] = rank
547
+ for call in compat_itervalues(function.calls):
548
+ if call.callee_id != function.id:
549
+ callee = self.functions[call.callee_id]
550
+ if callee.cycle is cycle:
551
+ self._rank_cycle_function(cycle, callee, rank + 1, ranks)
552
+
553
+ def _call_ratios_cycle(self, cycle, function, ranks, call_ratios, visited):
554
+ if function not in visited:
555
+ visited.add(function)
556
+ for call in compat_itervalues(function.calls):
557
+ if call.callee_id != function.id:
558
+ callee = self.functions[call.callee_id]
559
+ if callee.cycle is cycle:
560
+ if ranks[callee] > ranks[function]:
561
+ call_ratios[callee] = call_ratios.get(callee, 0.0) + call.ratio
562
+ self._call_ratios_cycle(cycle, callee, ranks, call_ratios, visited)
563
+
564
+ def _integrate_cycle_function(self, cycle, function, partial_ratio, partials, ranks, call_ratios, outevent, inevent):
565
+ if function not in partials:
566
+ partial = partial_ratio*function[inevent]
567
+ for call in compat_itervalues(function.calls):
568
+ if call.callee_id != function.id:
569
+ callee = self.functions[call.callee_id]
570
+ if callee.cycle is not cycle:
571
+ assert outevent in call
572
+ partial += partial_ratio*call[outevent]
573
+ else:
574
+ if ranks[callee] > ranks[function]:
575
+ callee_partial = self._integrate_cycle_function(cycle, callee, partial_ratio, partials, ranks, call_ratios, outevent, inevent)
576
+ call_ratio = ratio(call.ratio, call_ratios[callee])
577
+ call_partial = call_ratio*callee_partial
578
+ try:
579
+ call[outevent] += call_partial
580
+ except UndefinedEvent:
581
+ call[outevent] = call_partial
582
+ partial += call_partial
583
+ partials[function] = partial
584
+ try:
585
+ function[outevent] += partial
586
+ except UndefinedEvent:
587
+ function[outevent] = partial
588
+ return partials[function]
589
+
590
+ def aggregate(self, event):
591
+ """Aggregate an event for the whole profile."""
592
+
593
+ total = event.null()
594
+ for function in compat_itervalues(self.functions):
595
+ try:
596
+ total = event.aggregate(total, function[event])
597
+ except UndefinedEvent:
598
+ return
599
+ self[event] = total
600
+
601
+ def ratio(self, outevent, inevent):
602
+ assert outevent not in self
603
+ assert inevent in self
604
+ for function in compat_itervalues(self.functions):
605
+ assert outevent not in function
606
+ assert inevent in function
607
+ function[outevent] = ratio(function[inevent], self[inevent])
608
+ for call in compat_itervalues(function.calls):
609
+ assert outevent not in call
610
+ if inevent in call:
611
+ call[outevent] = ratio(call[inevent], self[inevent])
612
+ self[outevent] = 1.0
613
+
614
+ def prune(self, node_thres, edge_thres):
615
+ """Prune the profile"""
616
+
617
+ # compute the prune ratios
618
+ for function in compat_itervalues(self.functions):
619
+ try:
620
+ function.weight = function[TOTAL_TIME_RATIO]
621
+ except UndefinedEvent:
622
+ pass
623
+
624
+ for call in compat_itervalues(function.calls):
625
+ callee = self.functions[call.callee_id]
626
+
627
+ if TOTAL_TIME_RATIO in call:
628
+ # handle exact cases first
629
+ call.weight = call[TOTAL_TIME_RATIO]
630
+ else:
631
+ try:
632
+ # make a safe estimate
633
+ call.weight = min(function[TOTAL_TIME_RATIO], callee[TOTAL_TIME_RATIO])
634
+ except UndefinedEvent:
635
+ pass
636
+
637
+ # prune the nodes
638
+ for function_id in compat_keys(self.functions):
639
+ function = self.functions[function_id]
640
+ if function.weight is not None:
641
+ if function.weight < node_thres:
642
+ del self.functions[function_id]
643
+
644
+ # prune the egdes
645
+ for function in compat_itervalues(self.functions):
646
+ for callee_id in compat_keys(function.calls):
647
+ call = function.calls[callee_id]
648
+ if callee_id not in self.functions or call.weight is not None and call.weight < edge_thres:
649
+ del function.calls[callee_id]
650
+
651
+ def dump(self):
652
+ for function in compat_itervalues(self.functions):
653
+ sys.stderr.write('Function %s:\n' % (function.name,))
654
+ self._dump_events(function.events)
655
+ for call in compat_itervalues(function.calls):
656
+ callee = self.functions[call.callee_id]
657
+ sys.stderr.write(' Call %s:\n' % (callee.name,))
658
+ self._dump_events(call.events)
659
+ for cycle in self.cycles:
660
+ sys.stderr.write('Cycle:\n')
661
+ self._dump_events(cycle.events)
662
+ for function in cycle.functions:
663
+ sys.stderr.write(' Function %s\n' % (function.name,))
664
+
665
+ def _dump_events(self, events):
666
+ for event, value in compat_iteritems(events):
667
+ sys.stderr.write(' %s: %s\n' % (event.name, event.format(value)))
668
+
669
+
670
+ class Struct:
671
+ """Masquerade a dictionary with a structure-like behavior."""
672
+
673
+ def __init__(self, attrs = None):
674
+ if attrs is None:
675
+ attrs = {}
676
+ self.__dict__['_attrs'] = attrs
677
+
678
+ def __getattr__(self, name):
679
+ try:
680
+ return self._attrs[name]
681
+ except KeyError:
682
+ raise AttributeError(name)
683
+
684
+ def __setattr__(self, name, value):
685
+ self._attrs[name] = value
686
+
687
+ def __str__(self):
688
+ return str(self._attrs)
689
+
690
+ def __repr__(self):
691
+ return repr(self._attrs)
692
+
693
+
694
+ class ParseError(Exception):
695
+ """Raised when parsing to signal mismatches."""
696
+
697
+ def __init__(self, msg, line):
698
+ self.msg = msg
699
+ # TODO: store more source line information
700
+ self.line = line
701
+
702
+ def __str__(self):
703
+ return '%s: %r' % (self.msg, self.line)
704
+
705
+
706
+ class Parser:
707
+ """Parser interface."""
708
+
709
+ stdinInput = True
710
+ multipleInput = False
711
+
712
+ def __init__(self):
713
+ pass
714
+
715
+ def parse(self):
716
+ raise NotImplementedError
717
+
718
+
719
+ class LineParser(Parser):
720
+ """Base class for parsers that read line-based formats."""
721
+
722
+ def __init__(self, stream):
723
+ Parser.__init__(self)
724
+ self._stream = stream
725
+ self.__line = None
726
+ self.__eof = False
727
+ self.line_no = 0
728
+
729
+ def readline(self):
730
+ line = self._stream.readline()
731
+ if not line:
732
+ self.__line = ''
733
+ self.__eof = True
734
+ else:
735
+ self.line_no += 1
736
+ line = line.rstrip('\r\n')
737
+ if not PYTHON_3:
738
+ encoding = self._stream.encoding
739
+ if encoding is None:
740
+ encoding = locale.getpreferredencoding()
741
+ line = line.decode(encoding)
742
+ self.__line = line
743
+
744
+ def lookahead(self):
745
+ assert self.__line is not None
746
+ return self.__line
747
+
748
+ def consume(self):
749
+ assert self.__line is not None
750
+ line = self.__line
751
+ self.readline()
752
+ return line
753
+
754
+ def eof(self):
755
+ assert self.__line is not None
756
+ return self.__eof
757
+
758
+
759
+ XML_ELEMENT_START, XML_ELEMENT_END, XML_CHARACTER_DATA, XML_EOF = range(4)
760
+
761
+
762
+ class XmlToken:
763
+
764
+ def __init__(self, type, name_or_data, attrs = None, line = None, column = None):
765
+ assert type in (XML_ELEMENT_START, XML_ELEMENT_END, XML_CHARACTER_DATA, XML_EOF)
766
+ self.type = type
767
+ self.name_or_data = name_or_data
768
+ self.attrs = attrs
769
+ self.line = line
770
+ self.column = column
771
+
772
+ def __str__(self):
773
+ if self.type == XML_ELEMENT_START:
774
+ return '<' + self.name_or_data + ' ...>'
775
+ if self.type == XML_ELEMENT_END:
776
+ return '</' + self.name_or_data + '>'
777
+ if self.type == XML_CHARACTER_DATA:
778
+ return self.name_or_data
779
+ if self.type == XML_EOF:
780
+ return 'end of file'
781
+ assert 0
782
+
783
+
784
+ class XmlTokenizer:
785
+ """Expat based XML tokenizer."""
786
+
787
+ def __init__(self, fp, skip_ws = True):
788
+ self.fp = fp
789
+ self.tokens = []
790
+ self.index = 0
791
+ self.final = False
792
+ self.skip_ws = skip_ws
793
+
794
+ self.character_pos = 0, 0
795
+ self.character_data = ''
796
+
797
+ self.parser = xml.parsers.expat.ParserCreate()
798
+ self.parser.StartElementHandler = self.handle_element_start
799
+ self.parser.EndElementHandler = self.handle_element_end
800
+ self.parser.CharacterDataHandler = self.handle_character_data
801
+
802
+ def handle_element_start(self, name, attributes):
803
+ self.finish_character_data()
804
+ line, column = self.pos()
805
+ token = XmlToken(XML_ELEMENT_START, name, attributes, line, column)
806
+ self.tokens.append(token)
807
+
808
+ def handle_element_end(self, name):
809
+ self.finish_character_data()
810
+ line, column = self.pos()
811
+ token = XmlToken(XML_ELEMENT_END, name, None, line, column)
812
+ self.tokens.append(token)
813
+
814
+ def handle_character_data(self, data):
815
+ if not self.character_data:
816
+ self.character_pos = self.pos()
817
+ self.character_data += data
818
+
819
+ def finish_character_data(self):
820
+ if self.character_data:
821
+ if not self.skip_ws or not self.character_data.isspace():
822
+ line, column = self.character_pos
823
+ token = XmlToken(XML_CHARACTER_DATA, self.character_data, None, line, column)
824
+ self.tokens.append(token)
825
+ self.character_data = ''
826
+
827
+ def next(self):
828
+ size = 16*1024
829
+ while self.index >= len(self.tokens) and not self.final:
830
+ self.tokens = []
831
+ self.index = 0
832
+ data = self.fp.read(size)
833
+ self.final = len(data) < size
834
+ try:
835
+ self.parser.Parse(data, self.final)
836
+ except xml.parsers.expat.ExpatError as e:
837
+ #if e.code == xml.parsers.expat.errors.XML_ERROR_NO_ELEMENTS:
838
+ if e.code == 3:
839
+ pass
840
+ else:
841
+ raise e
842
+ if self.index >= len(self.tokens):
843
+ line, column = self.pos()
844
+ token = XmlToken(XML_EOF, None, None, line, column)
845
+ else:
846
+ token = self.tokens[self.index]
847
+ self.index += 1
848
+ return token
849
+
850
+ def pos(self):
851
+ return self.parser.CurrentLineNumber, self.parser.CurrentColumnNumber
852
+
853
+
854
+ class XmlTokenMismatch(Exception):
855
+
856
+ def __init__(self, expected, found):
857
+ self.expected = expected
858
+ self.found = found
859
+
860
+ def __str__(self):
861
+ return '%u:%u: %s expected, %s found' % (self.found.line, self.found.column, str(self.expected), str(self.found))
862
+
863
+
864
+ class XmlParser(Parser):
865
+ """Base XML document parser."""
866
+
867
+ def __init__(self, fp):
868
+ Parser.__init__(self)
869
+ self.tokenizer = XmlTokenizer(fp)
870
+ self.consume()
871
+
872
+ def consume(self):
873
+ self.token = self.tokenizer.next()
874
+
875
+ def match_element_start(self, name):
876
+ return self.token.type == XML_ELEMENT_START and self.token.name_or_data == name
877
+
878
+ def match_element_end(self, name):
879
+ return self.token.type == XML_ELEMENT_END and self.token.name_or_data == name
880
+
881
+ def element_start(self, name):
882
+ while self.token.type == XML_CHARACTER_DATA:
883
+ self.consume()
884
+ if self.token.type != XML_ELEMENT_START:
885
+ raise XmlTokenMismatch(XmlToken(XML_ELEMENT_START, name), self.token)
886
+ if self.token.name_or_data != name:
887
+ raise XmlTokenMismatch(XmlToken(XML_ELEMENT_START, name), self.token)
888
+ attrs = self.token.attrs
889
+ self.consume()
890
+ return attrs
891
+
892
+ def element_end(self, name):
893
+ while self.token.type == XML_CHARACTER_DATA:
894
+ self.consume()
895
+ if self.token.type != XML_ELEMENT_END:
896
+ raise XmlTokenMismatch(XmlToken(XML_ELEMENT_END, name), self.token)
897
+ if self.token.name_or_data != name:
898
+ raise XmlTokenMismatch(XmlToken(XML_ELEMENT_END, name), self.token)
899
+ self.consume()
900
+
901
+ def character_data(self, strip = True):
902
+ data = ''
903
+ while self.token.type == XML_CHARACTER_DATA:
904
+ data += self.token.name_or_data
905
+ self.consume()
906
+ if strip:
907
+ data = data.strip()
908
+ return data
909
+
910
+
911
+ class GprofParser(Parser):
912
+ """Parser for GNU gprof output.
913
+
914
+ See also:
915
+ - Chapter "Interpreting gprof's Output" from the GNU gprof manual
916
+ http://sourceware.org/binutils/docs-2.18/gprof/Call-Graph.html#Call-Graph
917
+ - File "cg_print.c" from the GNU gprof source code
918
+ http://sourceware.org/cgi-bin/cvsweb.cgi/~checkout~/src/gprof/cg_print.c?rev=1.12&cvsroot=src
919
+ """
920
+
921
+ def __init__(self, fp):
922
+ Parser.__init__(self)
923
+ self.fp = fp
924
+ self.functions = {}
925
+ self.cycles = {}
926
+
927
+ def readline(self):
928
+ line = self.fp.readline()
929
+ if not line:
930
+ sys.stderr.write('error: unexpected end of file\n')
931
+ sys.exit(1)
932
+ line = line.rstrip('\r\n')
933
+ return line
934
+
935
+ _int_re = re.compile(r'^\d+$')
936
+ _float_re = re.compile(r'^\d+\.\d+$')
937
+
938
+ def translate(self, mo):
939
+ """Extract a structure from a match object, while translating the types in the process."""
940
+ attrs = {}
941
+ groupdict = mo.groupdict()
942
+ for name, value in compat_iteritems(groupdict):
943
+ if value is None:
944
+ value = None
945
+ elif self._int_re.match(value):
946
+ value = int(value)
947
+ elif self._float_re.match(value):
948
+ value = float(value)
949
+ attrs[name] = (value)
950
+ return Struct(attrs)
951
+
952
+ _cg_header_re = re.compile(
953
+ # original gprof header
954
+ r'^\s+called/total\s+parents\s*$|' +
955
+ r'^index\s+%time\s+self\s+descendents\s+called\+self\s+name\s+index\s*$|' +
956
+ r'^\s+called/total\s+children\s*$|' +
957
+ # GNU gprof header
958
+ r'^index\s+%\s+time\s+self\s+children\s+called\s+name\s*$'
959
+ )
960
+
961
+ _cg_ignore_re = re.compile(
962
+ # spontaneous
963
+ r'^\s+<spontaneous>\s*$|'
964
+ # internal calls (such as "mcount")
965
+ r'^.*\((\d+)\)$'
966
+ )
967
+
968
+ _cg_primary_re = re.compile(
969
+ r'^\[(?P<index>\d+)\]?' +
970
+ r'\s+(?P<percentage_time>\d+\.\d+)' +
971
+ r'\s+(?P<self>\d+\.\d+)' +
972
+ r'\s+(?P<descendants>\d+\.\d+)' +
973
+ r'\s+(?:(?P<called>\d+)(?:\+(?P<called_self>\d+))?)?' +
974
+ r'\s+(?P<name>\S.*?)' +
975
+ r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +
976
+ r'\s\[(\d+)\]$'
977
+ )
978
+
979
+ _cg_parent_re = re.compile(
980
+ r'^\s+(?P<self>\d+\.\d+)?' +
981
+ r'\s+(?P<descendants>\d+\.\d+)?' +
982
+ r'\s+(?P<called>\d+)(?:/(?P<called_total>\d+))?' +
983
+ r'\s+(?P<name>\S.*?)' +
984
+ r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +
985
+ r'\s\[(?P<index>\d+)\]$'
986
+ )
987
+
988
+ _cg_child_re = _cg_parent_re
989
+
990
+ _cg_cycle_header_re = re.compile(
991
+ r'^\[(?P<index>\d+)\]?' +
992
+ r'\s+(?P<percentage_time>\d+\.\d+)' +
993
+ r'\s+(?P<self>\d+\.\d+)' +
994
+ r'\s+(?P<descendants>\d+\.\d+)' +
995
+ r'\s+(?:(?P<called>\d+)(?:\+(?P<called_self>\d+))?)?' +
996
+ r'\s+<cycle\s(?P<cycle>\d+)\sas\sa\swhole>' +
997
+ r'\s\[(\d+)\]$'
998
+ )
999
+
1000
+ _cg_cycle_member_re = re.compile(
1001
+ r'^\s+(?P<self>\d+\.\d+)?' +
1002
+ r'\s+(?P<descendants>\d+\.\d+)?' +
1003
+ r'\s+(?P<called>\d+)(?:\+(?P<called_self>\d+))?' +
1004
+ r'\s+(?P<name>\S.*?)' +
1005
+ r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +
1006
+ r'\s\[(?P<index>\d+)\]$'
1007
+ )
1008
+
1009
+ _cg_sep_re = re.compile(r'^--+$')
1010
+
1011
+ def parse_function_entry(self, lines):
1012
+ parents = []
1013
+ children = []
1014
+
1015
+ while True:
1016
+ if not lines:
1017
+ sys.stderr.write('warning: unexpected end of entry\n')
1018
+ line = lines.pop(0)
1019
+ if line.startswith('['):
1020
+ break
1021
+
1022
+ # read function parent line
1023
+ mo = self._cg_parent_re.match(line)
1024
+ if not mo:
1025
+ if self._cg_ignore_re.match(line):
1026
+ continue
1027
+ sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line)
1028
+ else:
1029
+ parent = self.translate(mo)
1030
+ parents.append(parent)
1031
+
1032
+ # read primary line
1033
+ mo = self._cg_primary_re.match(line)
1034
+ if not mo:
1035
+ sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line)
1036
+ return
1037
+ else:
1038
+ function = self.translate(mo)
1039
+
1040
+ while lines:
1041
+ line = lines.pop(0)
1042
+
1043
+ # read function subroutine line
1044
+ mo = self._cg_child_re.match(line)
1045
+ if not mo:
1046
+ if self._cg_ignore_re.match(line):
1047
+ continue
1048
+ sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line)
1049
+ else:
1050
+ child = self.translate(mo)
1051
+ children.append(child)
1052
+
1053
+ function.parents = parents
1054
+ function.children = children
1055
+
1056
+ self.functions[function.index] = function
1057
+
1058
+ def parse_cycle_entry(self, lines):
1059
+
1060
+ # read cycle header line
1061
+ line = lines[0]
1062
+ mo = self._cg_cycle_header_re.match(line)
1063
+ if not mo:
1064
+ sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line)
1065
+ return
1066
+ cycle = self.translate(mo)
1067
+
1068
+ # read cycle member lines
1069
+ cycle.functions = []
1070
+ for line in lines[1:]:
1071
+ mo = self._cg_cycle_member_re.match(line)
1072
+ if not mo:
1073
+ sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line)
1074
+ continue
1075
+ call = self.translate(mo)
1076
+ cycle.functions.append(call)
1077
+
1078
+ self.cycles[cycle.cycle] = cycle
1079
+
1080
+ def parse_cg_entry(self, lines):
1081
+ if lines[0].startswith("["):
1082
+ self.parse_cycle_entry(lines)
1083
+ else:
1084
+ self.parse_function_entry(lines)
1085
+
1086
+ def parse_cg(self):
1087
+ """Parse the call graph."""
1088
+
1089
+ # skip call graph header
1090
+ while not self._cg_header_re.match(self.readline()):
1091
+ pass
1092
+ line = self.readline()
1093
+ while self._cg_header_re.match(line):
1094
+ line = self.readline()
1095
+
1096
+ # process call graph entries
1097
+ entry_lines = []
1098
+ while line != '\014': # form feed
1099
+ if line and not line.isspace():
1100
+ if self._cg_sep_re.match(line):
1101
+ self.parse_cg_entry(entry_lines)
1102
+ entry_lines = []
1103
+ else:
1104
+ entry_lines.append(line)
1105
+ line = self.readline()
1106
+
1107
+ def parse(self):
1108
+ self.parse_cg()
1109
+ self.fp.close()
1110
+
1111
+ profile = Profile()
1112
+ profile[TIME] = 0.0
1113
+
1114
+ cycles = {}
1115
+ for index in self.cycles:
1116
+ cycles[index] = Cycle()
1117
+
1118
+ for entry in compat_itervalues(self.functions):
1119
+ # populate the function
1120
+ function = Function(entry.index, entry.name)
1121
+ function[TIME] = entry.self
1122
+ if entry.called is not None:
1123
+ function.called = entry.called
1124
+ if entry.called_self is not None:
1125
+ call = Call(entry.index)
1126
+ call[CALLS] = entry.called_self
1127
+ function.called += entry.called_self
1128
+
1129
+ # populate the function calls
1130
+ for child in entry.children:
1131
+ call = Call(child.index)
1132
+
1133
+ assert child.called is not None
1134
+ call[CALLS] = child.called
1135
+
1136
+ if child.index not in self.functions:
1137
+ # NOTE: functions that were never called but were discovered by gprof's
1138
+ # static call graph analysis dont have a call graph entry so we need
1139
+ # to add them here
1140
+ missing = Function(child.index, child.name)
1141
+ function[TIME] = 0.0
1142
+ function.called = 0
1143
+ profile.add_function(missing)
1144
+
1145
+ function.add_call(call)
1146
+
1147
+ profile.add_function(function)
1148
+
1149
+ if entry.cycle is not None:
1150
+ try:
1151
+ cycle = cycles[entry.cycle]
1152
+ except KeyError:
1153
+ sys.stderr.write('warning: <cycle %u as a whole> entry missing\n' % entry.cycle)
1154
+ cycle = Cycle()
1155
+ cycles[entry.cycle] = cycle
1156
+ cycle.add_function(function)
1157
+
1158
+ profile[TIME] = profile[TIME] + function[TIME]
1159
+
1160
+ for cycle in compat_itervalues(cycles):
1161
+ profile.add_cycle(cycle)
1162
+
1163
+ # Compute derived events
1164
+ profile.validate()
1165
+ profile.ratio(TIME_RATIO, TIME)
1166
+ profile.call_ratios(CALLS)
1167
+ profile.integrate(TOTAL_TIME, TIME)
1168
+ profile.ratio(TOTAL_TIME_RATIO, TOTAL_TIME)
1169
+
1170
+ return profile
1171
+
1172
+
1173
+ # Clone&hack of GprofParser for VTune Amplifier XE 2013 gprof-cc output.
1174
+ # Tested only with AXE 2013 for Windows.
1175
+ # - Use total times as reported by AXE.
1176
+ # - In the absence of call counts, call ratios are faked from the relative
1177
+ # proportions of total time. This affects only the weighting of the calls.
1178
+ # - Different header, separator, and end marker.
1179
+ # - Extra whitespace after function names.
1180
+ # - You get a full entry for <spontaneous>, which does not have parents.
1181
+ # - Cycles do have parents. These are saved but unused (as they are
1182
+ # for functions).
1183
+ # - Disambiguated "unrecognized call graph entry" error messages.
1184
+ # Notes:
1185
+ # - Total time of functions as reported by AXE passes the val3 test.
1186
+ # - CPU Time:Children in the input is sometimes a negative number. This
1187
+ # value goes to the variable descendants, which is unused.
1188
+ # - The format of gprof-cc reports is unaffected by the use of
1189
+ # -knob enable-call-counts=true (no call counts, ever), or
1190
+ # -show-as=samples (results are quoted in seconds regardless).
1191
+ class AXEParser(Parser):
1192
+ "Parser for VTune Amplifier XE 2013 gprof-cc report output."
1193
+
1194
+ def __init__(self, fp):
1195
+ Parser.__init__(self)
1196
+ self.fp = fp
1197
+ self.functions = {}
1198
+ self.cycles = {}
1199
+
1200
+ def readline(self):
1201
+ line = self.fp.readline()
1202
+ if not line:
1203
+ sys.stderr.write('error: unexpected end of file\n')
1204
+ sys.exit(1)
1205
+ line = line.rstrip('\r\n')
1206
+ return line
1207
+
1208
+ _int_re = re.compile(r'^\d+$')
1209
+ _float_re = re.compile(r'^\d+\.\d+$')
1210
+
1211
+ def translate(self, mo):
1212
+ """Extract a structure from a match object, while translating the types in the process."""
1213
+ attrs = {}
1214
+ groupdict = mo.groupdict()
1215
+ for name, value in compat_iteritems(groupdict):
1216
+ if value is None:
1217
+ value = None
1218
+ elif self._int_re.match(value):
1219
+ value = int(value)
1220
+ elif self._float_re.match(value):
1221
+ value = float(value)
1222
+ attrs[name] = (value)
1223
+ return Struct(attrs)
1224
+
1225
+ _cg_header_re = re.compile(
1226
+ '^Index |'
1227
+ '^-----+ '
1228
+ )
1229
+
1230
+ _cg_footer_re = re.compile('^Index\s+Function\s*$')
1231
+
1232
+ _cg_primary_re = re.compile(
1233
+ r'^\[(?P<index>\d+)\]?' +
1234
+ r'\s+(?P<percentage_time>\d+\.\d+)' +
1235
+ r'\s+(?P<self>\d+\.\d+)' +
1236
+ r'\s+(?P<descendants>\d+\.\d+)' +
1237
+ r'\s+(?P<name>\S.*?)' +
1238
+ r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +
1239
+ r'\s+\[(\d+)\]$'
1240
+ )
1241
+
1242
+ _cg_parent_re = re.compile(
1243
+ r'^\s+(?P<self>\d+\.\d+)?' +
1244
+ r'\s+(?P<descendants>\d+\.\d+)?' +
1245
+ r'\s+(?P<name>\S.*?)' +
1246
+ r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +
1247
+ r'\s+\[(?P<index>\d+)\]$'
1248
+ )
1249
+
1250
+ _cg_child_re = _cg_parent_re
1251
+
1252
+ _cg_cycle_header_re = re.compile(
1253
+ r'^\[(?P<index>\d+)\]?' +
1254
+ r'\s+(?P<percentage_time>\d+\.\d+)' +
1255
+ r'\s+(?P<self>\d+\.\d+)' +
1256
+ r'\s+(?P<descendants>\d+\.\d+)' +
1257
+ r'\s+<cycle\s(?P<cycle>\d+)\sas\sa\swhole>' +
1258
+ r'\s+\[(\d+)\]$'
1259
+ )
1260
+
1261
+ _cg_cycle_member_re = re.compile(
1262
+ r'^\s+(?P<self>\d+\.\d+)?' +
1263
+ r'\s+(?P<descendants>\d+\.\d+)?' +
1264
+ r'\s+(?P<name>\S.*?)' +
1265
+ r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +
1266
+ r'\s+\[(?P<index>\d+)\]$'
1267
+ )
1268
+
1269
+ def parse_function_entry(self, lines):
1270
+ parents = []
1271
+ children = []
1272
+
1273
+ while True:
1274
+ if not lines:
1275
+ sys.stderr.write('warning: unexpected end of entry\n')
1276
+ return
1277
+ line = lines.pop(0)
1278
+ if line.startswith('['):
1279
+ break
1280
+
1281
+ # read function parent line
1282
+ mo = self._cg_parent_re.match(line)
1283
+ if not mo:
1284
+ sys.stderr.write('warning: unrecognized call graph entry (1): %r\n' % line)
1285
+ else:
1286
+ parent = self.translate(mo)
1287
+ if parent.name != '<spontaneous>':
1288
+ parents.append(parent)
1289
+
1290
+ # read primary line
1291
+ mo = self._cg_primary_re.match(line)
1292
+ if not mo:
1293
+ sys.stderr.write('warning: unrecognized call graph entry (2): %r\n' % line)
1294
+ return
1295
+ else:
1296
+ function = self.translate(mo)
1297
+
1298
+ while lines:
1299
+ line = lines.pop(0)
1300
+
1301
+ # read function subroutine line
1302
+ mo = self._cg_child_re.match(line)
1303
+ if not mo:
1304
+ sys.stderr.write('warning: unrecognized call graph entry (3): %r\n' % line)
1305
+ else:
1306
+ child = self.translate(mo)
1307
+ if child.name != '<spontaneous>':
1308
+ children.append(child)
1309
+
1310
+ if function.name != '<spontaneous>':
1311
+ function.parents = parents
1312
+ function.children = children
1313
+
1314
+ self.functions[function.index] = function
1315
+
1316
+ def parse_cycle_entry(self, lines):
1317
+
1318
+ # Process the parents that were not there in gprof format.
1319
+ parents = []
1320
+ while True:
1321
+ if not lines:
1322
+ sys.stderr.write('warning: unexpected end of cycle entry\n')
1323
+ return
1324
+ line = lines.pop(0)
1325
+ if line.startswith('['):
1326
+ break
1327
+ mo = self._cg_parent_re.match(line)
1328
+ if not mo:
1329
+ sys.stderr.write('warning: unrecognized call graph entry (6): %r\n' % line)
1330
+ else:
1331
+ parent = self.translate(mo)
1332
+ if parent.name != '<spontaneous>':
1333
+ parents.append(parent)
1334
+
1335
+ # read cycle header line
1336
+ mo = self._cg_cycle_header_re.match(line)
1337
+ if not mo:
1338
+ sys.stderr.write('warning: unrecognized call graph entry (4): %r\n' % line)
1339
+ return
1340
+ cycle = self.translate(mo)
1341
+
1342
+ # read cycle member lines
1343
+ cycle.functions = []
1344
+ for line in lines[1:]:
1345
+ mo = self._cg_cycle_member_re.match(line)
1346
+ if not mo:
1347
+ sys.stderr.write('warning: unrecognized call graph entry (5): %r\n' % line)
1348
+ continue
1349
+ call = self.translate(mo)
1350
+ cycle.functions.append(call)
1351
+
1352
+ cycle.parents = parents
1353
+ self.cycles[cycle.cycle] = cycle
1354
+
1355
+ def parse_cg_entry(self, lines):
1356
+ if any("as a whole" in linelooper for linelooper in lines):
1357
+ self.parse_cycle_entry(lines)
1358
+ else:
1359
+ self.parse_function_entry(lines)
1360
+
1361
+ def parse_cg(self):
1362
+ """Parse the call graph."""
1363
+
1364
+ # skip call graph header
1365
+ line = self.readline()
1366
+ while self._cg_header_re.match(line):
1367
+ line = self.readline()
1368
+
1369
+ # process call graph entries
1370
+ entry_lines = []
1371
+ # An EOF in readline terminates the program without returning.
1372
+ while not self._cg_footer_re.match(line):
1373
+ if line.isspace():
1374
+ self.parse_cg_entry(entry_lines)
1375
+ entry_lines = []
1376
+ else:
1377
+ entry_lines.append(line)
1378
+ line = self.readline()
1379
+
1380
+ def parse(self):
1381
+ sys.stderr.write('warning: for axe format, edge weights are unreliable estimates derived from\nfunction total times.\n')
1382
+ self.parse_cg()
1383
+ self.fp.close()
1384
+
1385
+ profile = Profile()
1386
+ profile[TIME] = 0.0
1387
+
1388
+ cycles = {}
1389
+ for index in self.cycles:
1390
+ cycles[index] = Cycle()
1391
+
1392
+ for entry in compat_itervalues(self.functions):
1393
+ # populate the function
1394
+ function = Function(entry.index, entry.name)
1395
+ function[TIME] = entry.self
1396
+ function[TOTAL_TIME_RATIO] = entry.percentage_time / 100.0
1397
+
1398
+ # populate the function calls
1399
+ for child in entry.children:
1400
+ call = Call(child.index)
1401
+ # The following bogus value affects only the weighting of
1402
+ # the calls.
1403
+ call[TOTAL_TIME_RATIO] = function[TOTAL_TIME_RATIO]
1404
+
1405
+ if child.index not in self.functions:
1406
+ # NOTE: functions that were never called but were discovered by gprof's
1407
+ # static call graph analysis dont have a call graph entry so we need
1408
+ # to add them here
1409
+ # FIXME: Is this applicable?
1410
+ missing = Function(child.index, child.name)
1411
+ function[TIME] = 0.0
1412
+ profile.add_function(missing)
1413
+
1414
+ function.add_call(call)
1415
+
1416
+ profile.add_function(function)
1417
+
1418
+ if entry.cycle is not None:
1419
+ try:
1420
+ cycle = cycles[entry.cycle]
1421
+ except KeyError:
1422
+ sys.stderr.write('warning: <cycle %u as a whole> entry missing\n' % entry.cycle)
1423
+ cycle = Cycle()
1424
+ cycles[entry.cycle] = cycle
1425
+ cycle.add_function(function)
1426
+
1427
+ profile[TIME] = profile[TIME] + function[TIME]
1428
+
1429
+ for cycle in compat_itervalues(cycles):
1430
+ profile.add_cycle(cycle)
1431
+
1432
+ # Compute derived events.
1433
+ profile.validate()
1434
+ profile.ratio(TIME_RATIO, TIME)
1435
+ # Lacking call counts, fake call ratios based on total times.
1436
+ profile.call_ratios(TOTAL_TIME_RATIO)
1437
+ # The TOTAL_TIME_RATIO of functions is already set. Propagate that
1438
+ # total time to the calls. (TOTAL_TIME is neither set nor used.)
1439
+ for function in compat_itervalues(profile.functions):
1440
+ for call in compat_itervalues(function.calls):
1441
+ if call.ratio is not None:
1442
+ callee = profile.functions[call.callee_id]
1443
+ call[TOTAL_TIME_RATIO] = call.ratio * callee[TOTAL_TIME_RATIO];
1444
+
1445
+ return profile
1446
+
1447
+
1448
+ class CallgrindParser(LineParser):
1449
+ """Parser for valgrind's callgrind tool.
1450
+
1451
+ See also:
1452
+ - http://valgrind.org/docs/manual/cl-format.html
1453
+ """
1454
+
1455
+ _call_re = re.compile('^calls=\s*(\d+)\s+((\d+|\+\d+|-\d+|\*)\s+)+$')
1456
+
1457
+ def __init__(self, infile):
1458
+ LineParser.__init__(self, infile)
1459
+
1460
+ # Textual positions
1461
+ self.position_ids = {}
1462
+ self.positions = {}
1463
+
1464
+ # Numeric positions
1465
+ self.num_positions = 1
1466
+ self.cost_positions = ['line']
1467
+ self.last_positions = [0]
1468
+
1469
+ # Events
1470
+ self.num_events = 0
1471
+ self.cost_events = []
1472
+
1473
+ self.profile = Profile()
1474
+ self.profile[SAMPLES] = 0
1475
+
1476
+ def parse(self):
1477
+ # read lookahead
1478
+ self.readline()
1479
+
1480
+ self.parse_key('version')
1481
+ self.parse_key('creator')
1482
+ while self.parse_part():
1483
+ pass
1484
+ if not self.eof():
1485
+ sys.stderr.write('warning: line %u: unexpected line\n' % self.line_no)
1486
+ sys.stderr.write('%s\n' % self.lookahead())
1487
+
1488
+ # compute derived data
1489
+ self.profile.validate()
1490
+ self.profile.find_cycles()
1491
+ self.profile.ratio(TIME_RATIO, SAMPLES)
1492
+ self.profile.call_ratios(CALLS)
1493
+ self.profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
1494
+
1495
+ return self.profile
1496
+
1497
+ def parse_part(self):
1498
+ if not self.parse_header_line():
1499
+ return False
1500
+ while self.parse_header_line():
1501
+ pass
1502
+ if not self.parse_body_line():
1503
+ return False
1504
+ while self.parse_body_line():
1505
+ pass
1506
+ return True
1507
+
1508
+ def parse_header_line(self):
1509
+ return \
1510
+ self.parse_empty() or \
1511
+ self.parse_comment() or \
1512
+ self.parse_part_detail() or \
1513
+ self.parse_description() or \
1514
+ self.parse_event_specification() or \
1515
+ self.parse_cost_line_def() or \
1516
+ self.parse_cost_summary()
1517
+
1518
+ _detail_keys = set(('cmd', 'pid', 'thread', 'part'))
1519
+
1520
+ def parse_part_detail(self):
1521
+ return self.parse_keys(self._detail_keys)
1522
+
1523
+ def parse_description(self):
1524
+ return self.parse_key('desc') is not None
1525
+
1526
+ def parse_event_specification(self):
1527
+ event = self.parse_key('event')
1528
+ if event is None:
1529
+ return False
1530
+ return True
1531
+
1532
+ def parse_cost_line_def(self):
1533
+ pair = self.parse_keys(('events', 'positions'))
1534
+ if pair is None:
1535
+ return False
1536
+ key, value = pair
1537
+ items = value.split()
1538
+ if key == 'events':
1539
+ self.num_events = len(items)
1540
+ self.cost_events = items
1541
+ if key == 'positions':
1542
+ self.num_positions = len(items)
1543
+ self.cost_positions = items
1544
+ self.last_positions = [0]*self.num_positions
1545
+ return True
1546
+
1547
+ def parse_cost_summary(self):
1548
+ pair = self.parse_keys(('summary', 'totals'))
1549
+ if pair is None:
1550
+ return False
1551
+ return True
1552
+
1553
+ def parse_body_line(self):
1554
+ return \
1555
+ self.parse_empty() or \
1556
+ self.parse_comment() or \
1557
+ self.parse_cost_line() or \
1558
+ self.parse_position_spec() or \
1559
+ self.parse_association_spec()
1560
+
1561
+ __subpos_re = r'(0x[0-9a-fA-F]+|\d+|\+\d+|-\d+|\*)'
1562
+ _cost_re = re.compile(r'^' +
1563
+ __subpos_re + r'( +' + __subpos_re + r')*' +
1564
+ r'( +\d+)*' +
1565
+ '$')
1566
+
1567
+ def parse_cost_line(self, calls=None):
1568
+ line = self.lookahead().rstrip()
1569
+ mo = self._cost_re.match(line)
1570
+ if not mo:
1571
+ return False
1572
+
1573
+ function = self.get_function()
1574
+
1575
+ if calls is None:
1576
+ # Unlike other aspects, call object (cob) is relative not to the
1577
+ # last call object, but to the caller's object (ob), so try to
1578
+ # update it when processing a functions cost line
1579
+ try:
1580
+ self.positions['cob'] = self.positions['ob']
1581
+ except KeyError:
1582
+ pass
1583
+
1584
+ values = line.split()
1585
+ assert len(values) <= self.num_positions + self.num_events
1586
+
1587
+ positions = values[0 : self.num_positions]
1588
+ events = values[self.num_positions : ]
1589
+ events += ['0']*(self.num_events - len(events))
1590
+
1591
+ for i in range(self.num_positions):
1592
+ position = positions[i]
1593
+ if position == '*':
1594
+ position = self.last_positions[i]
1595
+ elif position[0] in '-+':
1596
+ position = self.last_positions[i] + int(position)
1597
+ elif position.startswith('0x'):
1598
+ position = int(position, 16)
1599
+ else:
1600
+ position = int(position)
1601
+ self.last_positions[i] = position
1602
+
1603
+ events = [float(event) for event in events]
1604
+
1605
+ if calls is None:
1606
+ function[SAMPLES] += events[0]
1607
+ self.profile[SAMPLES] += events[0]
1608
+ else:
1609
+ callee = self.get_callee()
1610
+ callee.called += calls
1611
+
1612
+ try:
1613
+ call = function.calls[callee.id]
1614
+ except KeyError:
1615
+ call = Call(callee.id)
1616
+ call[CALLS] = calls
1617
+ call[SAMPLES] = events[0]
1618
+ function.add_call(call)
1619
+ else:
1620
+ call[CALLS] += calls
1621
+ call[SAMPLES] += events[0]
1622
+
1623
+ self.consume()
1624
+ return True
1625
+
1626
+ def parse_association_spec(self):
1627
+ line = self.lookahead()
1628
+ if not line.startswith('calls='):
1629
+ return False
1630
+
1631
+ _, values = line.split('=', 1)
1632
+ values = values.strip().split()
1633
+ calls = int(values[0])
1634
+ call_position = values[1:]
1635
+ self.consume()
1636
+
1637
+ self.parse_cost_line(calls)
1638
+
1639
+ return True
1640
+
1641
+ _position_re = re.compile('^(?P<position>[cj]?(?:ob|fl|fi|fe|fn))=\s*(?:\((?P<id>\d+)\))?(?:\s*(?P<name>.+))?')
1642
+
1643
+ _position_table_map = {
1644
+ 'ob': 'ob',
1645
+ 'fl': 'fl',
1646
+ 'fi': 'fl',
1647
+ 'fe': 'fl',
1648
+ 'fn': 'fn',
1649
+ 'cob': 'ob',
1650
+ 'cfl': 'fl',
1651
+ 'cfi': 'fl',
1652
+ 'cfe': 'fl',
1653
+ 'cfn': 'fn',
1654
+ 'jfi': 'fl',
1655
+ }
1656
+
1657
+ _position_map = {
1658
+ 'ob': 'ob',
1659
+ 'fl': 'fl',
1660
+ 'fi': 'fl',
1661
+ 'fe': 'fl',
1662
+ 'fn': 'fn',
1663
+ 'cob': 'cob',
1664
+ 'cfl': 'cfl',
1665
+ 'cfi': 'cfl',
1666
+ 'cfe': 'cfl',
1667
+ 'cfn': 'cfn',
1668
+ 'jfi': 'jfi',
1669
+ }
1670
+
1671
+ def parse_position_spec(self):
1672
+ line = self.lookahead()
1673
+
1674
+ if line.startswith('jump=') or line.startswith('jcnd='):
1675
+ self.consume()
1676
+ return True
1677
+
1678
+ mo = self._position_re.match(line)
1679
+ if not mo:
1680
+ return False
1681
+
1682
+ position, id, name = mo.groups()
1683
+ if id:
1684
+ table = self._position_table_map[position]
1685
+ if name:
1686
+ self.position_ids[(table, id)] = name
1687
+ else:
1688
+ name = self.position_ids.get((table, id), '')
1689
+ self.positions[self._position_map[position]] = name
1690
+
1691
+ self.consume()
1692
+ return True
1693
+
1694
+ def parse_empty(self):
1695
+ if self.eof():
1696
+ return False
1697
+ line = self.lookahead()
1698
+ if line.strip():
1699
+ return False
1700
+ self.consume()
1701
+ return True
1702
+
1703
+ def parse_comment(self):
1704
+ line = self.lookahead()
1705
+ if not line.startswith('#'):
1706
+ return False
1707
+ self.consume()
1708
+ return True
1709
+
1710
+ _key_re = re.compile(r'^(\w+):')
1711
+
1712
+ def parse_key(self, key):
1713
+ pair = self.parse_keys((key,))
1714
+ if not pair:
1715
+ return None
1716
+ key, value = pair
1717
+ return value
1718
+ line = self.lookahead()
1719
+ mo = self._key_re.match(line)
1720
+ if not mo:
1721
+ return None
1722
+ key, value = line.split(':', 1)
1723
+ if key not in keys:
1724
+ return None
1725
+ value = value.strip()
1726
+ self.consume()
1727
+ return key, value
1728
+
1729
+ def parse_keys(self, keys):
1730
+ line = self.lookahead()
1731
+ mo = self._key_re.match(line)
1732
+ if not mo:
1733
+ return None
1734
+ key, value = line.split(':', 1)
1735
+ if key not in keys:
1736
+ return None
1737
+ value = value.strip()
1738
+ self.consume()
1739
+ return key, value
1740
+
1741
+ def make_function(self, module, filename, name):
1742
+ # FIXME: module and filename are not being tracked reliably
1743
+ #id = '|'.join((module, filename, name))
1744
+ id = name
1745
+ try:
1746
+ function = self.profile.functions[id]
1747
+ except KeyError:
1748
+ function = Function(id, name)
1749
+ if module:
1750
+ function.module = os.path.basename(module)
1751
+ function[SAMPLES] = 0
1752
+ function.called = 0
1753
+ self.profile.add_function(function)
1754
+ return function
1755
+
1756
+ def get_function(self):
1757
+ module = self.positions.get('ob', '')
1758
+ filename = self.positions.get('fl', '')
1759
+ function = self.positions.get('fn', '')
1760
+ return self.make_function(module, filename, function)
1761
+
1762
+ def get_callee(self):
1763
+ module = self.positions.get('cob', '')
1764
+ filename = self.positions.get('cfi', '')
1765
+ function = self.positions.get('cfn', '')
1766
+ return self.make_function(module, filename, function)
1767
+
1768
+
1769
+ class PerfParser(LineParser):
1770
+ """Parser for linux perf callgraph output.
1771
+
1772
+ It expects output generated with
1773
+
1774
+ perf record -g
1775
+ perf script | gprof2dot.py --format=perf
1776
+ """
1777
+
1778
+ def __init__(self, infile):
1779
+ LineParser.__init__(self, infile)
1780
+ self.profile = Profile()
1781
+
1782
+ def readline(self):
1783
+ # Override LineParser.readline to ignore comment lines
1784
+ while True:
1785
+ LineParser.readline(self)
1786
+ if self.eof() or not self.lookahead().startswith('#'):
1787
+ break
1788
+
1789
+ def parse(self):
1790
+ # read lookahead
1791
+ self.readline()
1792
+
1793
+ profile = self.profile
1794
+ profile[SAMPLES] = 0
1795
+ while not self.eof():
1796
+ self.parse_event()
1797
+
1798
+ # compute derived data
1799
+ profile.validate()
1800
+ profile.find_cycles()
1801
+ profile.ratio(TIME_RATIO, SAMPLES)
1802
+ profile.call_ratios(SAMPLES2)
1803
+ if totalMethod == "callratios":
1804
+ # Heuristic approach. TOTAL_SAMPLES is unused.
1805
+ profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
1806
+ elif totalMethod == "callstacks":
1807
+ # Use the actual call chains for functions.
1808
+ profile[TOTAL_SAMPLES] = profile[SAMPLES]
1809
+ profile.ratio(TOTAL_TIME_RATIO, TOTAL_SAMPLES)
1810
+ # Then propagate that total time to the calls.
1811
+ for function in compat_itervalues(profile.functions):
1812
+ for call in compat_itervalues(function.calls):
1813
+ if call.ratio is not None:
1814
+ callee = profile.functions[call.callee_id]
1815
+ call[TOTAL_TIME_RATIO] = call.ratio * callee[TOTAL_TIME_RATIO];
1816
+ else:
1817
+ assert False
1818
+
1819
+ return profile
1820
+
1821
+ def parse_event(self):
1822
+ if self.eof():
1823
+ return
1824
+
1825
+ line = self.consume()
1826
+ assert line
1827
+
1828
+ callchain = self.parse_callchain()
1829
+ if not callchain:
1830
+ return
1831
+
1832
+ callee = callchain[0]
1833
+ callee[SAMPLES] += 1
1834
+ self.profile[SAMPLES] += 1
1835
+
1836
+ for caller in callchain[1:]:
1837
+ try:
1838
+ call = caller.calls[callee.id]
1839
+ except KeyError:
1840
+ call = Call(callee.id)
1841
+ call[SAMPLES2] = 1
1842
+ caller.add_call(call)
1843
+ else:
1844
+ call[SAMPLES2] += 1
1845
+
1846
+ callee = caller
1847
+
1848
+ # Increment TOTAL_SAMPLES only once on each function.
1849
+ stack = set(callchain)
1850
+ for function in stack:
1851
+ function[TOTAL_SAMPLES] += 1
1852
+
1853
+ def parse_callchain(self):
1854
+ callchain = []
1855
+ while self.lookahead():
1856
+ function = self.parse_call()
1857
+ if function is None:
1858
+ break
1859
+ callchain.append(function)
1860
+ if self.lookahead() == '':
1861
+ self.consume()
1862
+ return callchain
1863
+
1864
+ call_re = re.compile(r'^\s+(?P<address>[0-9a-fA-F]+)\s+(?P<symbol>.*)\s+\((?P<module>[^)]*)\)$')
1865
+
1866
+ def parse_call(self):
1867
+ line = self.consume()
1868
+ mo = self.call_re.match(line)
1869
+ assert mo
1870
+ if not mo:
1871
+ return None
1872
+
1873
+ function_name = mo.group('symbol')
1874
+ if not function_name:
1875
+ function_name = mo.group('address')
1876
+
1877
+ module = mo.group('module')
1878
+
1879
+ function_id = function_name + ':' + module
1880
+
1881
+ try:
1882
+ function = self.profile.functions[function_id]
1883
+ except KeyError:
1884
+ function = Function(function_id, function_name)
1885
+ function.module = os.path.basename(module)
1886
+ function[SAMPLES] = 0
1887
+ function[TOTAL_SAMPLES] = 0
1888
+ self.profile.add_function(function)
1889
+
1890
+ return function
1891
+
1892
+
1893
+ class OprofileParser(LineParser):
1894
+ """Parser for oprofile callgraph output.
1895
+
1896
+ See also:
1897
+ - http://oprofile.sourceforge.net/doc/opreport.html#opreport-callgraph
1898
+ """
1899
+
1900
+ _fields_re = {
1901
+ 'samples': r'(\d+)',
1902
+ '%': r'(\S+)',
1903
+ 'linenr info': r'(?P<source>\(no location information\)|\S+:\d+)',
1904
+ 'image name': r'(?P<image>\S+(?:\s\(tgid:[^)]*\))?)',
1905
+ 'app name': r'(?P<application>\S+)',
1906
+ 'symbol name': r'(?P<symbol>\(no symbols\)|.+?)',
1907
+ }
1908
+
1909
+ def __init__(self, infile):
1910
+ LineParser.__init__(self, infile)
1911
+ self.entries = {}
1912
+ self.entry_re = None
1913
+
1914
+ def add_entry(self, callers, function, callees):
1915
+ try:
1916
+ entry = self.entries[function.id]
1917
+ except KeyError:
1918
+ self.entries[function.id] = (callers, function, callees)
1919
+ else:
1920
+ callers_total, function_total, callees_total = entry
1921
+ self.update_subentries_dict(callers_total, callers)
1922
+ function_total.samples += function.samples
1923
+ self.update_subentries_dict(callees_total, callees)
1924
+
1925
+ def update_subentries_dict(self, totals, partials):
1926
+ for partial in compat_itervalues(partials):
1927
+ try:
1928
+ total = totals[partial.id]
1929
+ except KeyError:
1930
+ totals[partial.id] = partial
1931
+ else:
1932
+ total.samples += partial.samples
1933
+
1934
+ def parse(self):
1935
+ # read lookahead
1936
+ self.readline()
1937
+
1938
+ self.parse_header()
1939
+ while self.lookahead():
1940
+ self.parse_entry()
1941
+
1942
+ profile = Profile()
1943
+
1944
+ reverse_call_samples = {}
1945
+
1946
+ # populate the profile
1947
+ profile[SAMPLES] = 0
1948
+ for _callers, _function, _callees in compat_itervalues(self.entries):
1949
+ function = Function(_function.id, _function.name)
1950
+ function[SAMPLES] = _function.samples
1951
+ profile.add_function(function)
1952
+ profile[SAMPLES] += _function.samples
1953
+
1954
+ if _function.application:
1955
+ function.process = os.path.basename(_function.application)
1956
+ if _function.image:
1957
+ function.module = os.path.basename(_function.image)
1958
+
1959
+ total_callee_samples = 0
1960
+ for _callee in compat_itervalues(_callees):
1961
+ total_callee_samples += _callee.samples
1962
+
1963
+ for _callee in compat_itervalues(_callees):
1964
+ if not _callee.self:
1965
+ call = Call(_callee.id)
1966
+ call[SAMPLES2] = _callee.samples
1967
+ function.add_call(call)
1968
+
1969
+ # compute derived data
1970
+ profile.validate()
1971
+ profile.find_cycles()
1972
+ profile.ratio(TIME_RATIO, SAMPLES)
1973
+ profile.call_ratios(SAMPLES2)
1974
+ profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
1975
+
1976
+ return profile
1977
+
1978
+ def parse_header(self):
1979
+ while not self.match_header():
1980
+ self.consume()
1981
+ line = self.lookahead()
1982
+ fields = re.split(r'\s\s+', line)
1983
+ entry_re = r'^\s*' + r'\s+'.join([self._fields_re[field] for field in fields]) + r'(?P<self>\s+\[self\])?$'
1984
+ self.entry_re = re.compile(entry_re)
1985
+ self.skip_separator()
1986
+
1987
+ def parse_entry(self):
1988
+ callers = self.parse_subentries()
1989
+ if self.match_primary():
1990
+ function = self.parse_subentry()
1991
+ if function is not None:
1992
+ callees = self.parse_subentries()
1993
+ self.add_entry(callers, function, callees)
1994
+ self.skip_separator()
1995
+
1996
+ def parse_subentries(self):
1997
+ subentries = {}
1998
+ while self.match_secondary():
1999
+ subentry = self.parse_subentry()
2000
+ subentries[subentry.id] = subentry
2001
+ return subentries
2002
+
2003
+ def parse_subentry(self):
2004
+ entry = Struct()
2005
+ line = self.consume()
2006
+ mo = self.entry_re.match(line)
2007
+ if not mo:
2008
+ raise ParseError('failed to parse', line)
2009
+ fields = mo.groupdict()
2010
+ entry.samples = int(mo.group(1))
2011
+ if 'source' in fields and fields['source'] != '(no location information)':
2012
+ source = fields['source']
2013
+ filename, lineno = source.split(':')
2014
+ entry.filename = filename
2015
+ entry.lineno = int(lineno)
2016
+ else:
2017
+ source = ''
2018
+ entry.filename = None
2019
+ entry.lineno = None
2020
+ entry.image = fields.get('image', '')
2021
+ entry.application = fields.get('application', '')
2022
+ if 'symbol' in fields and fields['symbol'] != '(no symbols)':
2023
+ entry.symbol = fields['symbol']
2024
+ else:
2025
+ entry.symbol = ''
2026
+ if entry.symbol.startswith('"') and entry.symbol.endswith('"'):
2027
+ entry.symbol = entry.symbol[1:-1]
2028
+ entry.id = ':'.join((entry.application, entry.image, source, entry.symbol))
2029
+ entry.self = fields.get('self', None) != None
2030
+ if entry.self:
2031
+ entry.id += ':self'
2032
+ if entry.symbol:
2033
+ entry.name = entry.symbol
2034
+ else:
2035
+ entry.name = entry.image
2036
+ return entry
2037
+
2038
+ def skip_separator(self):
2039
+ while not self.match_separator():
2040
+ self.consume()
2041
+ self.consume()
2042
+
2043
+ def match_header(self):
2044
+ line = self.lookahead()
2045
+ return line.startswith('samples')
2046
+
2047
+ def match_separator(self):
2048
+ line = self.lookahead()
2049
+ return line == '-'*len(line)
2050
+
2051
+ def match_primary(self):
2052
+ line = self.lookahead()
2053
+ return not line[:1].isspace()
2054
+
2055
+ def match_secondary(self):
2056
+ line = self.lookahead()
2057
+ return line[:1].isspace()
2058
+
2059
+
2060
+ class HProfParser(LineParser):
2061
+ """Parser for java hprof output
2062
+
2063
+ See also:
2064
+ - http://java.sun.com/developer/technicalArticles/Programming/HPROF.html
2065
+ """
2066
+
2067
+ trace_re = re.compile(r'\t(.*)\((.*):(.*)\)')
2068
+ trace_id_re = re.compile(r'^TRACE (\d+):$')
2069
+
2070
+ def __init__(self, infile):
2071
+ LineParser.__init__(self, infile)
2072
+ self.traces = {}
2073
+ self.samples = {}
2074
+
2075
+ def parse(self):
2076
+ # read lookahead
2077
+ self.readline()
2078
+
2079
+ while not self.lookahead().startswith('------'): self.consume()
2080
+ while not self.lookahead().startswith('TRACE '): self.consume()
2081
+
2082
+ self.parse_traces()
2083
+
2084
+ while not self.lookahead().startswith('CPU'):
2085
+ self.consume()
2086
+
2087
+ self.parse_samples()
2088
+
2089
+ # populate the profile
2090
+ profile = Profile()
2091
+ profile[SAMPLES] = 0
2092
+
2093
+ functions = {}
2094
+
2095
+ # build up callgraph
2096
+ for id, trace in compat_iteritems(self.traces):
2097
+ if not id in self.samples: continue
2098
+ mtime = self.samples[id][0]
2099
+ last = None
2100
+
2101
+ for func, file, line in trace:
2102
+ if not func in functions:
2103
+ function = Function(func, func)
2104
+ function[SAMPLES] = 0
2105
+ profile.add_function(function)
2106
+ functions[func] = function
2107
+
2108
+ function = functions[func]
2109
+ # allocate time to the deepest method in the trace
2110
+ if not last:
2111
+ function[SAMPLES] += mtime
2112
+ profile[SAMPLES] += mtime
2113
+ else:
2114
+ c = function.get_call(last)
2115
+ c[SAMPLES2] += mtime
2116
+
2117
+ last = func
2118
+
2119
+ # compute derived data
2120
+ profile.validate()
2121
+ profile.find_cycles()
2122
+ profile.ratio(TIME_RATIO, SAMPLES)
2123
+ profile.call_ratios(SAMPLES2)
2124
+ profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
2125
+
2126
+ return profile
2127
+
2128
+ def parse_traces(self):
2129
+ while self.lookahead().startswith('TRACE '):
2130
+ self.parse_trace()
2131
+
2132
+ def parse_trace(self):
2133
+ l = self.consume()
2134
+ mo = self.trace_id_re.match(l)
2135
+ tid = mo.group(1)
2136
+ last = None
2137
+ trace = []
2138
+
2139
+ while self.lookahead().startswith('\t'):
2140
+ l = self.consume()
2141
+ match = self.trace_re.search(l)
2142
+ if not match:
2143
+ #sys.stderr.write('Invalid line: %s\n' % l)
2144
+ break
2145
+ else:
2146
+ function_name, file, line = match.groups()
2147
+ trace += [(function_name, file, line)]
2148
+
2149
+ self.traces[int(tid)] = trace
2150
+
2151
+ def parse_samples(self):
2152
+ self.consume()
2153
+ self.consume()
2154
+
2155
+ while not self.lookahead().startswith('CPU'):
2156
+ rank, percent_self, percent_accum, count, traceid, method = self.lookahead().split()
2157
+ self.samples[int(traceid)] = (int(count), method)
2158
+ self.consume()
2159
+
2160
+
2161
+ class SysprofParser(XmlParser):
2162
+
2163
+ def __init__(self, stream):
2164
+ XmlParser.__init__(self, stream)
2165
+
2166
+ def parse(self):
2167
+ objects = {}
2168
+ nodes = {}
2169
+
2170
+ self.element_start('profile')
2171
+ while self.token.type == XML_ELEMENT_START:
2172
+ if self.token.name_or_data == 'objects':
2173
+ assert not objects
2174
+ objects = self.parse_items('objects')
2175
+ elif self.token.name_or_data == 'nodes':
2176
+ assert not nodes
2177
+ nodes = self.parse_items('nodes')
2178
+ else:
2179
+ self.parse_value(self.token.name_or_data)
2180
+ self.element_end('profile')
2181
+
2182
+ return self.build_profile(objects, nodes)
2183
+
2184
+ def parse_items(self, name):
2185
+ assert name[-1] == 's'
2186
+ items = {}
2187
+ self.element_start(name)
2188
+ while self.token.type == XML_ELEMENT_START:
2189
+ id, values = self.parse_item(name[:-1])
2190
+ assert id not in items
2191
+ items[id] = values
2192
+ self.element_end(name)
2193
+ return items
2194
+
2195
+ def parse_item(self, name):
2196
+ attrs = self.element_start(name)
2197
+ id = int(attrs['id'])
2198
+ values = self.parse_values()
2199
+ self.element_end(name)
2200
+ return id, values
2201
+
2202
+ def parse_values(self):
2203
+ values = {}
2204
+ while self.token.type == XML_ELEMENT_START:
2205
+ name = self.token.name_or_data
2206
+ value = self.parse_value(name)
2207
+ assert name not in values
2208
+ values[name] = value
2209
+ return values
2210
+
2211
+ def parse_value(self, tag):
2212
+ self.element_start(tag)
2213
+ value = self.character_data()
2214
+ self.element_end(tag)
2215
+ if value.isdigit():
2216
+ return int(value)
2217
+ if value.startswith('"') and value.endswith('"'):
2218
+ return value[1:-1]
2219
+ return value
2220
+
2221
+ def build_profile(self, objects, nodes):
2222
+ profile = Profile()
2223
+
2224
+ profile[SAMPLES] = 0
2225
+ for id, object in compat_iteritems(objects):
2226
+ # Ignore fake objects (process names, modules, "Everything", "kernel", etc.)
2227
+ if object['self'] == 0:
2228
+ continue
2229
+
2230
+ function = Function(id, object['name'])
2231
+ function[SAMPLES] = object['self']
2232
+ profile.add_function(function)
2233
+ profile[SAMPLES] += function[SAMPLES]
2234
+
2235
+ for id, node in compat_iteritems(nodes):
2236
+ # Ignore fake calls
2237
+ if node['self'] == 0:
2238
+ continue
2239
+
2240
+ # Find a non-ignored parent
2241
+ parent_id = node['parent']
2242
+ while parent_id != 0:
2243
+ parent = nodes[parent_id]
2244
+ caller_id = parent['object']
2245
+ if objects[caller_id]['self'] != 0:
2246
+ break
2247
+ parent_id = parent['parent']
2248
+ if parent_id == 0:
2249
+ continue
2250
+
2251
+ callee_id = node['object']
2252
+
2253
+ assert objects[caller_id]['self']
2254
+ assert objects[callee_id]['self']
2255
+
2256
+ function = profile.functions[caller_id]
2257
+
2258
+ samples = node['self']
2259
+ try:
2260
+ call = function.calls[callee_id]
2261
+ except KeyError:
2262
+ call = Call(callee_id)
2263
+ call[SAMPLES2] = samples
2264
+ function.add_call(call)
2265
+ else:
2266
+ call[SAMPLES2] += samples
2267
+
2268
+ # Compute derived events
2269
+ profile.validate()
2270
+ profile.find_cycles()
2271
+ profile.ratio(TIME_RATIO, SAMPLES)
2272
+ profile.call_ratios(SAMPLES2)
2273
+ profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
2274
+
2275
+ return profile
2276
+
2277
+
2278
+ class XPerfParser(Parser):
2279
+ """Parser for CSVs generted by XPerf, from Microsoft Windows Performance Tools.
2280
+ """
2281
+
2282
+ def __init__(self, stream):
2283
+ Parser.__init__(self)
2284
+ self.stream = stream
2285
+ self.profile = Profile()
2286
+ self.profile[SAMPLES] = 0
2287
+ self.column = {}
2288
+
2289
+ def parse(self):
2290
+ import csv
2291
+ reader = csv.reader(
2292
+ self.stream,
2293
+ delimiter = ',',
2294
+ quotechar = None,
2295
+ escapechar = None,
2296
+ doublequote = False,
2297
+ skipinitialspace = True,
2298
+ lineterminator = '\r\n',
2299
+ quoting = csv.QUOTE_NONE)
2300
+ header = True
2301
+ for row in reader:
2302
+ if header:
2303
+ self.parse_header(row)
2304
+ header = False
2305
+ else:
2306
+ self.parse_row(row)
2307
+
2308
+ # compute derived data
2309
+ self.profile.validate()
2310
+ self.profile.find_cycles()
2311
+ self.profile.ratio(TIME_RATIO, SAMPLES)
2312
+ self.profile.call_ratios(SAMPLES2)
2313
+ self.profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
2314
+
2315
+ return self.profile
2316
+
2317
+ def parse_header(self, row):
2318
+ for column in range(len(row)):
2319
+ name = row[column]
2320
+ assert name not in self.column
2321
+ self.column[name] = column
2322
+
2323
+ def parse_row(self, row):
2324
+ fields = {}
2325
+ for name, column in compat_iteritems(self.column):
2326
+ value = row[column]
2327
+ for factory in int, float:
2328
+ try:
2329
+ value = factory(value)
2330
+ except ValueError:
2331
+ pass
2332
+ else:
2333
+ break
2334
+ fields[name] = value
2335
+
2336
+ process = fields['Process Name']
2337
+ symbol = fields['Module'] + '!' + fields['Function']
2338
+ weight = fields['Weight']
2339
+ count = fields['Count']
2340
+
2341
+ if process == 'Idle':
2342
+ return
2343
+
2344
+ function = self.get_function(process, symbol)
2345
+ function[SAMPLES] += weight * count
2346
+ self.profile[SAMPLES] += weight * count
2347
+
2348
+ stack = fields['Stack']
2349
+ if stack != '?':
2350
+ stack = stack.split('/')
2351
+ assert stack[0] == '[Root]'
2352
+ if stack[-1] != symbol:
2353
+ # XXX: some cases the sampled function does not appear in the stack
2354
+ stack.append(symbol)
2355
+ caller = None
2356
+ for symbol in stack[1:]:
2357
+ callee = self.get_function(process, symbol)
2358
+ if caller is not None:
2359
+ try:
2360
+ call = caller.calls[callee.id]
2361
+ except KeyError:
2362
+ call = Call(callee.id)
2363
+ call[SAMPLES2] = count
2364
+ caller.add_call(call)
2365
+ else:
2366
+ call[SAMPLES2] += count
2367
+ caller = callee
2368
+
2369
+ def get_function(self, process, symbol):
2370
+ function_id = process + '!' + symbol
2371
+
2372
+ try:
2373
+ function = self.profile.functions[function_id]
2374
+ except KeyError:
2375
+ module, name = symbol.split('!', 1)
2376
+ function = Function(function_id, name)
2377
+ function.process = process
2378
+ function.module = module
2379
+ function[SAMPLES] = 0
2380
+ self.profile.add_function(function)
2381
+
2382
+ return function
2383
+
2384
+
2385
+ class SleepyParser(Parser):
2386
+ """Parser for GNU gprof output.
2387
+
2388
+ See also:
2389
+ - http://www.codersnotes.com/sleepy/
2390
+ - http://sleepygraph.sourceforge.net/
2391
+ """
2392
+
2393
+ stdinInput = False
2394
+
2395
+ def __init__(self, filename):
2396
+ Parser.__init__(self)
2397
+
2398
+ from zipfile import ZipFile
2399
+
2400
+ self.database = ZipFile(filename)
2401
+
2402
+ self.symbols = {}
2403
+ self.calls = {}
2404
+
2405
+ self.profile = Profile()
2406
+
2407
+ _symbol_re = re.compile(
2408
+ r'^(?P<id>\w+)' +
2409
+ r'\s+"(?P<module>[^"]*)"' +
2410
+ r'\s+"(?P<procname>[^"]*)"' +
2411
+ r'\s+"(?P<sourcefile>[^"]*)"' +
2412
+ r'\s+(?P<sourceline>\d+)$'
2413
+ )
2414
+
2415
+ def openEntry(self, name):
2416
+ # Some versions of verysleepy use lowercase filenames
2417
+ for database_name in self.database.namelist():
2418
+ if name.lower() == database_name.lower():
2419
+ name = database_name
2420
+ break
2421
+
2422
+ return self.database.open(name, 'rU')
2423
+
2424
+ def parse_symbols(self):
2425
+ for line in self.openEntry('Symbols.txt'):
2426
+ line = line.decode('UTF-8')
2427
+
2428
+ mo = self._symbol_re.match(line)
2429
+ if mo:
2430
+ symbol_id, module, procname, sourcefile, sourceline = mo.groups()
2431
+
2432
+ function_id = ':'.join([module, procname])
2433
+
2434
+ try:
2435
+ function = self.profile.functions[function_id]
2436
+ except KeyError:
2437
+ function = Function(function_id, procname)
2438
+ function.module = module
2439
+ function[SAMPLES] = 0
2440
+ self.profile.add_function(function)
2441
+
2442
+ self.symbols[symbol_id] = function
2443
+
2444
+ def parse_callstacks(self):
2445
+ for line in self.openEntry('Callstacks.txt'):
2446
+ line = line.decode('UTF-8')
2447
+
2448
+ fields = line.split()
2449
+ samples = float(fields[0])
2450
+ callstack = fields[1:]
2451
+
2452
+ callstack = [self.symbols[symbol_id] for symbol_id in callstack]
2453
+
2454
+ callee = callstack[0]
2455
+
2456
+ callee[SAMPLES] += samples
2457
+ self.profile[SAMPLES] += samples
2458
+
2459
+ for caller in callstack[1:]:
2460
+ try:
2461
+ call = caller.calls[callee.id]
2462
+ except KeyError:
2463
+ call = Call(callee.id)
2464
+ call[SAMPLES2] = samples
2465
+ caller.add_call(call)
2466
+ else:
2467
+ call[SAMPLES2] += samples
2468
+
2469
+ callee = caller
2470
+
2471
+ def parse(self):
2472
+ profile = self.profile
2473
+ profile[SAMPLES] = 0
2474
+
2475
+ self.parse_symbols()
2476
+ self.parse_callstacks()
2477
+
2478
+ # Compute derived events
2479
+ profile.validate()
2480
+ profile.find_cycles()
2481
+ profile.ratio(TIME_RATIO, SAMPLES)
2482
+ profile.call_ratios(SAMPLES2)
2483
+ profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
2484
+
2485
+ return profile
2486
+
2487
+
2488
+ class AQtimeTable:
2489
+
2490
+ def __init__(self, name, fields):
2491
+ self.name = name
2492
+
2493
+ self.fields = fields
2494
+ self.field_column = {}
2495
+ for column in range(len(fields)):
2496
+ self.field_column[fields[column]] = column
2497
+ self.rows = []
2498
+
2499
+ def __len__(self):
2500
+ return len(self.rows)
2501
+
2502
+ def __iter__(self):
2503
+ for values, children in self.rows:
2504
+ fields = {}
2505
+ for name, value in zip(self.fields, values):
2506
+ fields[name] = value
2507
+ children = dict([(child.name, child) for child in children])
2508
+ yield fields, children
2509
+ raise StopIteration
2510
+
2511
+ def add_row(self, values, children=()):
2512
+ self.rows.append((values, children))
2513
+
2514
+
2515
+ class AQtimeParser(XmlParser):
2516
+
2517
+ def __init__(self, stream):
2518
+ XmlParser.__init__(self, stream)
2519
+ self.tables = {}
2520
+
2521
+ def parse(self):
2522
+ self.element_start('AQtime_Results')
2523
+ self.parse_headers()
2524
+ results = self.parse_results()
2525
+ self.element_end('AQtime_Results')
2526
+ return self.build_profile(results)
2527
+
2528
+ def parse_headers(self):
2529
+ self.element_start('HEADERS')
2530
+ while self.token.type == XML_ELEMENT_START:
2531
+ self.parse_table_header()
2532
+ self.element_end('HEADERS')
2533
+
2534
+ def parse_table_header(self):
2535
+ attrs = self.element_start('TABLE_HEADER')
2536
+ name = attrs['NAME']
2537
+ id = int(attrs['ID'])
2538
+ field_types = []
2539
+ field_names = []
2540
+ while self.token.type == XML_ELEMENT_START:
2541
+ field_type, field_name = self.parse_table_field()
2542
+ field_types.append(field_type)
2543
+ field_names.append(field_name)
2544
+ self.element_end('TABLE_HEADER')
2545
+ self.tables[id] = name, field_types, field_names
2546
+
2547
+ def parse_table_field(self):
2548
+ attrs = self.element_start('TABLE_FIELD')
2549
+ type = attrs['TYPE']
2550
+ name = self.character_data()
2551
+ self.element_end('TABLE_FIELD')
2552
+ return type, name
2553
+
2554
+ def parse_results(self):
2555
+ self.element_start('RESULTS')
2556
+ table = self.parse_data()
2557
+ self.element_end('RESULTS')
2558
+ return table
2559
+
2560
+ def parse_data(self):
2561
+ rows = []
2562
+ attrs = self.element_start('DATA')
2563
+ table_id = int(attrs['TABLE_ID'])
2564
+ table_name, field_types, field_names = self.tables[table_id]
2565
+ table = AQtimeTable(table_name, field_names)
2566
+ while self.token.type == XML_ELEMENT_START:
2567
+ row, children = self.parse_row(field_types)
2568
+ table.add_row(row, children)
2569
+ self.element_end('DATA')
2570
+ return table
2571
+
2572
+ def parse_row(self, field_types):
2573
+ row = [None]*len(field_types)
2574
+ children = []
2575
+ self.element_start('ROW')
2576
+ while self.token.type == XML_ELEMENT_START:
2577
+ if self.token.name_or_data == 'FIELD':
2578
+ field_id, field_value = self.parse_field(field_types)
2579
+ row[field_id] = field_value
2580
+ elif self.token.name_or_data == 'CHILDREN':
2581
+ children = self.parse_children()
2582
+ else:
2583
+ raise XmlTokenMismatch("<FIELD ...> or <CHILDREN ...>", self.token)
2584
+ self.element_end('ROW')
2585
+ return row, children
2586
+
2587
+ def parse_field(self, field_types):
2588
+ attrs = self.element_start('FIELD')
2589
+ id = int(attrs['ID'])
2590
+ type = field_types[id]
2591
+ value = self.character_data()
2592
+ if type == 'Integer':
2593
+ value = int(value)
2594
+ elif type == 'Float':
2595
+ value = float(value)
2596
+ elif type == 'Address':
2597
+ value = int(value)
2598
+ elif type == 'String':
2599
+ pass
2600
+ else:
2601
+ assert False
2602
+ self.element_end('FIELD')
2603
+ return id, value
2604
+
2605
+ def parse_children(self):
2606
+ children = []
2607
+ self.element_start('CHILDREN')
2608
+ while self.token.type == XML_ELEMENT_START:
2609
+ table = self.parse_data()
2610
+ assert table.name not in children
2611
+ children.append(table)
2612
+ self.element_end('CHILDREN')
2613
+ return children
2614
+
2615
+ def build_profile(self, results):
2616
+ assert results.name == 'Routines'
2617
+ profile = Profile()
2618
+ profile[TIME] = 0.0
2619
+ for fields, tables in results:
2620
+ function = self.build_function(fields)
2621
+ children = tables['Children']
2622
+ for fields, _ in children:
2623
+ call = self.build_call(fields)
2624
+ function.add_call(call)
2625
+ profile.add_function(function)
2626
+ profile[TIME] = profile[TIME] + function[TIME]
2627
+ profile[TOTAL_TIME] = profile[TIME]
2628
+ profile.ratio(TOTAL_TIME_RATIO, TOTAL_TIME)
2629
+ return profile
2630
+
2631
+ def build_function(self, fields):
2632
+ function = Function(self.build_id(fields), self.build_name(fields))
2633
+ function[TIME] = fields['Time']
2634
+ function[TOTAL_TIME] = fields['Time with Children']
2635
+ #function[TIME_RATIO] = fields['% Time']/100.0
2636
+ #function[TOTAL_TIME_RATIO] = fields['% with Children']/100.0
2637
+ return function
2638
+
2639
+ def build_call(self, fields):
2640
+ call = Call(self.build_id(fields))
2641
+ call[TIME] = fields['Time']
2642
+ call[TOTAL_TIME] = fields['Time with Children']
2643
+ #call[TIME_RATIO] = fields['% Time']/100.0
2644
+ #call[TOTAL_TIME_RATIO] = fields['% with Children']/100.0
2645
+ return call
2646
+
2647
+ def build_id(self, fields):
2648
+ return ':'.join([fields['Module Name'], fields['Unit Name'], fields['Routine Name']])
2649
+
2650
+ def build_name(self, fields):
2651
+ # TODO: use more fields
2652
+ return fields['Routine Name']
2653
+
2654
+
2655
+ class PstatsParser:
2656
+ """Parser python profiling statistics saved with te pstats module."""
2657
+
2658
+ stdinInput = False
2659
+ multipleInput = True
2660
+
2661
+ def __init__(self, *filename):
2662
+ import pstats
2663
+ try:
2664
+ self.stats = pstats.Stats(*filename)
2665
+ except ValueError:
2666
+ if sys.version_info[0] >= 3:
2667
+ raise
2668
+ import hotshot.stats
2669
+ self.stats = hotshot.stats.load(filename[0])
2670
+ self.profile = Profile()
2671
+ self.function_ids = {}
2672
+
2673
+ def get_function_name(self, key):
2674
+ filename, line, name = key
2675
+ module = os.path.splitext(filename)[0]
2676
+ module = os.path.basename(module)
2677
+ return "%s:%d:%s" % (module, line, name)
2678
+
2679
+ def get_function(self, key):
2680
+ try:
2681
+ id = self.function_ids[key]
2682
+ except KeyError:
2683
+ id = len(self.function_ids)
2684
+ name = self.get_function_name(key)
2685
+ function = Function(id, name)
2686
+ self.profile.functions[id] = function
2687
+ self.function_ids[key] = id
2688
+ else:
2689
+ function = self.profile.functions[id]
2690
+ return function
2691
+
2692
+ def parse(self):
2693
+ self.profile[TIME] = 0.0
2694
+ self.profile[TOTAL_TIME] = self.stats.total_tt
2695
+ for fn, (cc, nc, tt, ct, callers) in compat_iteritems(self.stats.stats):
2696
+ callee = self.get_function(fn)
2697
+ callee.called = nc
2698
+ callee[TOTAL_TIME] = ct
2699
+ callee[TIME] = tt
2700
+ self.profile[TIME] += tt
2701
+ self.profile[TOTAL_TIME] = max(self.profile[TOTAL_TIME], ct)
2702
+ for fn, value in compat_iteritems(callers):
2703
+ caller = self.get_function(fn)
2704
+ call = Call(callee.id)
2705
+ if isinstance(value, tuple):
2706
+ for i in xrange(0, len(value), 4):
2707
+ nc, cc, tt, ct = value[i:i+4]
2708
+ if CALLS in call:
2709
+ call[CALLS] += cc
2710
+ else:
2711
+ call[CALLS] = cc
2712
+
2713
+ if TOTAL_TIME in call:
2714
+ call[TOTAL_TIME] += ct
2715
+ else:
2716
+ call[TOTAL_TIME] = ct
2717
+
2718
+ else:
2719
+ call[CALLS] = value
2720
+ call[TOTAL_TIME] = ratio(value, nc)*ct
2721
+
2722
+ caller.add_call(call)
2723
+ #self.stats.print_stats()
2724
+ #self.stats.print_callees()
2725
+
2726
+ # Compute derived events
2727
+ self.profile.validate()
2728
+ self.profile.ratio(TIME_RATIO, TIME)
2729
+ self.profile.ratio(TOTAL_TIME_RATIO, TOTAL_TIME)
2730
+
2731
+ return self.profile
2732
+
2733
+
2734
+ class Theme:
2735
+
2736
+ def __init__(self,
2737
+ bgcolor = (0.0, 0.0, 1.0),
2738
+ mincolor = (0.0, 0.0, 0.0),
2739
+ maxcolor = (0.0, 0.0, 1.0),
2740
+ fontname = "Arial",
2741
+ fontcolor = "white",
2742
+ nodestyle = "filled",
2743
+ minfontsize = 10.0,
2744
+ maxfontsize = 10.0,
2745
+ minpenwidth = 0.5,
2746
+ maxpenwidth = 4.0,
2747
+ gamma = 2.2,
2748
+ skew = 1.0):
2749
+ self.bgcolor = bgcolor
2750
+ self.mincolor = mincolor
2751
+ self.maxcolor = maxcolor
2752
+ self.fontname = fontname
2753
+ self.fontcolor = fontcolor
2754
+ self.nodestyle = nodestyle
2755
+ self.minfontsize = minfontsize
2756
+ self.maxfontsize = maxfontsize
2757
+ self.minpenwidth = minpenwidth
2758
+ self.maxpenwidth = maxpenwidth
2759
+ self.gamma = gamma
2760
+ self.skew = skew
2761
+
2762
+ def graph_bgcolor(self):
2763
+ return self.hsl_to_rgb(*self.bgcolor)
2764
+
2765
+ def graph_fontname(self):
2766
+ return self.fontname
2767
+
2768
+ def graph_fontcolor(self):
2769
+ return self.fontcolor
2770
+
2771
+ def graph_fontsize(self):
2772
+ return self.minfontsize
2773
+
2774
+ def node_bgcolor(self, weight):
2775
+ return self.color(weight)
2776
+
2777
+ def node_fgcolor(self, weight):
2778
+ if self.nodestyle == "filled":
2779
+ return self.graph_bgcolor()
2780
+ else:
2781
+ return self.color(weight)
2782
+
2783
+ def node_fontsize(self, weight):
2784
+ return self.fontsize(weight)
2785
+
2786
+ def node_style(self):
2787
+ return self.nodestyle
2788
+
2789
+ def edge_color(self, weight):
2790
+ return self.color(weight)
2791
+
2792
+ def edge_fontsize(self, weight):
2793
+ return self.fontsize(weight)
2794
+
2795
+ def edge_penwidth(self, weight):
2796
+ return max(weight*self.maxpenwidth, self.minpenwidth)
2797
+
2798
+ def edge_arrowsize(self, weight):
2799
+ return 0.5 * math.sqrt(self.edge_penwidth(weight))
2800
+
2801
+ def fontsize(self, weight):
2802
+ return max(weight**2 * self.maxfontsize, self.minfontsize)
2803
+
2804
+ def color(self, weight):
2805
+ weight = min(max(weight, 0.0), 1.0)
2806
+
2807
+ hmin, smin, lmin = self.mincolor
2808
+ hmax, smax, lmax = self.maxcolor
2809
+
2810
+ if self.skew < 0:
2811
+ raise ValueError("Skew must be greater than 0")
2812
+ elif self.skew == 1.0:
2813
+ h = hmin + weight*(hmax - hmin)
2814
+ s = smin + weight*(smax - smin)
2815
+ l = lmin + weight*(lmax - lmin)
2816
+ else:
2817
+ base = self.skew
2818
+ h = hmin + ((hmax-hmin)*(-1.0 + (base ** weight)) / (base - 1.0))
2819
+ s = smin + ((smax-smin)*(-1.0 + (base ** weight)) / (base - 1.0))
2820
+ l = lmin + ((lmax-lmin)*(-1.0 + (base ** weight)) / (base - 1.0))
2821
+
2822
+ return self.hsl_to_rgb(h, s, l)
2823
+
2824
+ def hsl_to_rgb(self, h, s, l):
2825
+ """Convert a color from HSL color-model to RGB.
2826
+
2827
+ See also:
2828
+ - http://www.w3.org/TR/css3-color/#hsl-color
2829
+ """
2830
+
2831
+ h = h % 1.0
2832
+ s = min(max(s, 0.0), 1.0)
2833
+ l = min(max(l, 0.0), 1.0)
2834
+
2835
+ if l <= 0.5:
2836
+ m2 = l*(s + 1.0)
2837
+ else:
2838
+ m2 = l + s - l*s
2839
+ m1 = l*2.0 - m2
2840
+ r = self._hue_to_rgb(m1, m2, h + 1.0/3.0)
2841
+ g = self._hue_to_rgb(m1, m2, h)
2842
+ b = self._hue_to_rgb(m1, m2, h - 1.0/3.0)
2843
+
2844
+ # Apply gamma correction
2845
+ r **= self.gamma
2846
+ g **= self.gamma
2847
+ b **= self.gamma
2848
+
2849
+ return (r, g, b)
2850
+
2851
+ def _hue_to_rgb(self, m1, m2, h):
2852
+ if h < 0.0:
2853
+ h += 1.0
2854
+ elif h > 1.0:
2855
+ h -= 1.0
2856
+ if h*6 < 1.0:
2857
+ return m1 + (m2 - m1)*h*6.0
2858
+ elif h*2 < 1.0:
2859
+ return m2
2860
+ elif h*3 < 2.0:
2861
+ return m1 + (m2 - m1)*(2.0/3.0 - h)*6.0
2862
+ else:
2863
+ return m1
2864
+
2865
+
2866
+ TEMPERATURE_COLORMAP = Theme(
2867
+ mincolor = (2.0/3.0, 0.80, 0.25), # dark blue
2868
+ maxcolor = (0.0, 1.0, 0.5), # satured red
2869
+ gamma = 1.0
2870
+ )
2871
+
2872
+ PINK_COLORMAP = Theme(
2873
+ mincolor = (0.0, 1.0, 0.90), # pink
2874
+ maxcolor = (0.0, 1.0, 0.5), # satured red
2875
+ )
2876
+
2877
+ GRAY_COLORMAP = Theme(
2878
+ mincolor = (0.0, 0.0, 0.85), # light gray
2879
+ maxcolor = (0.0, 0.0, 0.0), # black
2880
+ )
2881
+
2882
+ BW_COLORMAP = Theme(
2883
+ minfontsize = 8.0,
2884
+ maxfontsize = 24.0,
2885
+ mincolor = (0.0, 0.0, 0.0), # black
2886
+ maxcolor = (0.0, 0.0, 0.0), # black
2887
+ minpenwidth = 0.1,
2888
+ maxpenwidth = 8.0,
2889
+ )
2890
+
2891
+ PRINT_COLORMAP = Theme(
2892
+ minfontsize = 18.0,
2893
+ maxfontsize = 30.0,
2894
+ fontcolor = "black",
2895
+ nodestyle = "solid",
2896
+ mincolor = (0.0, 0.0, 0.0), # black
2897
+ maxcolor = (0.0, 0.0, 0.0), # black
2898
+ minpenwidth = 0.1,
2899
+ maxpenwidth = 8.0,
2900
+ )
2901
+
2902
+
2903
+ class DotWriter:
2904
+ """Writer for the DOT language.
2905
+
2906
+ See also:
2907
+ - "The DOT Language" specification
2908
+ http://www.graphviz.org/doc/info/lang.html
2909
+ """
2910
+
2911
+ strip = False
2912
+ wrap = False
2913
+
2914
+ def __init__(self, fp):
2915
+ self.fp = fp
2916
+
2917
+ def wrap_function_name(self, name):
2918
+ """Split the function name on multiple lines."""
2919
+
2920
+ if len(name) > 32:
2921
+ ratio = 2.0/3.0
2922
+ height = max(int(len(name)/(1.0 - ratio) + 0.5), 1)
2923
+ width = max(len(name)/height, 32)
2924
+ # TODO: break lines in symbols
2925
+ name = textwrap.fill(name, width, break_long_words=False)
2926
+
2927
+ # Take away spaces
2928
+ name = name.replace(", ", ",")
2929
+ name = name.replace("> >", ">>")
2930
+ name = name.replace("> >", ">>") # catch consecutive
2931
+
2932
+ return name
2933
+
2934
+ show_function_events = [TOTAL_TIME_RATIO, TIME_RATIO]
2935
+ show_edge_events = [TOTAL_TIME_RATIO, CALLS]
2936
+
2937
+ def graph(self, profile, theme):
2938
+ self.begin_graph()
2939
+
2940
+ fontname = theme.graph_fontname()
2941
+ fontcolor = theme.graph_fontcolor()
2942
+ nodestyle = theme.node_style()
2943
+
2944
+ self.attr('graph', fontname=fontname, ranksep=0.25, nodesep=0.125)
2945
+ self.attr('node', fontname=fontname, shape="box", style=nodestyle, fontcolor=fontcolor, width=0, height=0)
2946
+ self.attr('edge', fontname=fontname)
2947
+
2948
+ for function in compat_itervalues(profile.functions):
2949
+ labels = []
2950
+ if function.process is not None:
2951
+ labels.append(function.process)
2952
+ if function.module is not None:
2953
+ labels.append(function.module)
2954
+
2955
+ if self.strip:
2956
+ function_name = function.stripped_name()
2957
+ else:
2958
+ function_name = function.name
2959
+ if self.wrap:
2960
+ function_name = self.wrap_function_name(function_name)
2961
+ labels.append(function_name)
2962
+
2963
+ for event in self.show_function_events:
2964
+ if event in function.events:
2965
+ label = event.format(function[event])
2966
+ labels.append(label)
2967
+ if function.called is not None:
2968
+ labels.append("%u%s" % (function.called, MULTIPLICATION_SIGN))
2969
+
2970
+ if function.weight is not None:
2971
+ weight = function.weight
2972
+ else:
2973
+ weight = 0.0
2974
+
2975
+ label = '\n'.join(labels)
2976
+ self.node(function.id,
2977
+ label = label,
2978
+ color = self.color(theme.node_bgcolor(weight)),
2979
+ fontcolor = self.color(theme.node_fgcolor(weight)),
2980
+ fontsize = "%.2f" % theme.node_fontsize(weight),
2981
+ )
2982
+
2983
+ for call in compat_itervalues(function.calls):
2984
+ callee = profile.functions[call.callee_id]
2985
+
2986
+ labels = []
2987
+ for event in self.show_edge_events:
2988
+ if event in call.events:
2989
+ label = event.format(call[event])
2990
+ labels.append(label)
2991
+
2992
+ if call.weight is not None:
2993
+ weight = call.weight
2994
+ elif callee.weight is not None:
2995
+ weight = callee.weight
2996
+ else:
2997
+ weight = 0.0
2998
+
2999
+ label = '\n'.join(labels)
3000
+
3001
+ self.edge(function.id, call.callee_id,
3002
+ label = label,
3003
+ color = self.color(theme.edge_color(weight)),
3004
+ fontcolor = self.color(theme.edge_color(weight)),
3005
+ fontsize = "%.2f" % theme.edge_fontsize(weight),
3006
+ penwidth = "%.2f" % theme.edge_penwidth(weight),
3007
+ labeldistance = "%.2f" % theme.edge_penwidth(weight),
3008
+ arrowsize = "%.2f" % theme.edge_arrowsize(weight),
3009
+ )
3010
+
3011
+ self.end_graph()
3012
+
3013
+ def begin_graph(self):
3014
+ self.write('digraph {\n')
3015
+
3016
+ def end_graph(self):
3017
+ self.write('}\n')
3018
+
3019
+ def attr(self, what, **attrs):
3020
+ self.write("\t")
3021
+ self.write(what)
3022
+ self.attr_list(attrs)
3023
+ self.write(";\n")
3024
+
3025
+ def node(self, node, **attrs):
3026
+ self.write("\t")
3027
+ self.id(node)
3028
+ self.attr_list(attrs)
3029
+ self.write(";\n")
3030
+
3031
+ def edge(self, src, dst, **attrs):
3032
+ self.write("\t")
3033
+ self.id(src)
3034
+ self.write(" -> ")
3035
+ self.id(dst)
3036
+ self.attr_list(attrs)
3037
+ self.write(";\n")
3038
+
3039
+ def attr_list(self, attrs):
3040
+ if not attrs:
3041
+ return
3042
+ self.write(' [')
3043
+ first = True
3044
+ for name, value in compat_iteritems(attrs):
3045
+ if first:
3046
+ first = False
3047
+ else:
3048
+ self.write(", ")
3049
+ self.id(name)
3050
+ self.write('=')
3051
+ self.id(value)
3052
+ self.write(']')
3053
+
3054
+ def id(self, id):
3055
+ if isinstance(id, (int, float)):
3056
+ s = str(id)
3057
+ elif isinstance(id, basestring):
3058
+ if id.isalnum() and not id.startswith('0x'):
3059
+ s = id
3060
+ else:
3061
+ s = self.escape(id)
3062
+ else:
3063
+ raise TypeError
3064
+ self.write(s)
3065
+
3066
+ def color(self, rgb):
3067
+ r, g, b = rgb
3068
+
3069
+ def float2int(f):
3070
+ if f <= 0.0:
3071
+ return 0
3072
+ if f >= 1.0:
3073
+ return 255
3074
+ return int(255.0*f + 0.5)
3075
+
3076
+ return "#" + "".join(["%02x" % float2int(c) for c in (r, g, b)])
3077
+
3078
+ def escape(self, s):
3079
+ if not PYTHON_3:
3080
+ s = s.encode('utf-8')
3081
+ s = s.replace('\\', r'\\')
3082
+ s = s.replace('\n', r'\n')
3083
+ s = s.replace('\t', r'\t')
3084
+ s = s.replace('"', r'\"')
3085
+ return '"' + s + '"'
3086
+
3087
+ def write(self, s):
3088
+ self.fp.write(s)
3089
+
3090
+
3091
+ class Main:
3092
+ """Main program."""
3093
+
3094
+ themes = {
3095
+ "color": TEMPERATURE_COLORMAP,
3096
+ "pink": PINK_COLORMAP,
3097
+ "gray": GRAY_COLORMAP,
3098
+ "bw": BW_COLORMAP,
3099
+ "print": PRINT_COLORMAP,
3100
+ }
3101
+
3102
+ formats = {
3103
+ "aqtime": AQtimeParser,
3104
+ "axe": AXEParser,
3105
+ "callgrind": CallgrindParser,
3106
+ "hprof": HProfParser,
3107
+ "oprofile": OprofileParser,
3108
+ "perf": PerfParser,
3109
+ "prof": GprofParser,
3110
+ "pstats": PstatsParser,
3111
+ "sleepy": SleepyParser,
3112
+ "sysprof": SysprofParser,
3113
+ "xperf": XPerfParser,
3114
+ }
3115
+
3116
+ def naturalJoin(self, values):
3117
+ if len(values) >= 2:
3118
+ return ', '.join(values[:-1]) + ' or ' + values[-1]
3119
+
3120
+ else:
3121
+ return ''.join(values)
3122
+
3123
+ def main(self):
3124
+ """Main program."""
3125
+
3126
+ global totalMethod
3127
+
3128
+ formatNames = list(self.formats.keys())
3129
+ formatNames.sort()
3130
+
3131
+ optparser = optparse.OptionParser(
3132
+ usage="\n\t%prog [options] [file] ...")
3133
+ optparser.add_option(
3134
+ '-o', '--output', metavar='FILE',
3135
+ type="string", dest="output",
3136
+ help="output filename [stdout]")
3137
+ optparser.add_option(
3138
+ '-n', '--node-thres', metavar='PERCENTAGE',
3139
+ type="float", dest="node_thres", default=0.5,
3140
+ help="eliminate nodes below this threshold [default: %default]")
3141
+ optparser.add_option(
3142
+ '-e', '--edge-thres', metavar='PERCENTAGE',
3143
+ type="float", dest="edge_thres", default=0.1,
3144
+ help="eliminate edges below this threshold [default: %default]")
3145
+ optparser.add_option(
3146
+ '-f', '--format',
3147
+ type="choice", choices=formatNames,
3148
+ dest="format", default="prof",
3149
+ help="profile format: %s [default: %%default]" % self.naturalJoin(formatNames))
3150
+ optparser.add_option(
3151
+ '--total',
3152
+ type="choice", choices=('callratios', 'callstacks'),
3153
+ dest="totalMethod", default=totalMethod,
3154
+ help="preferred method of calculating total time: callratios or callstacks (currently affects only perf format) [default: %default]")
3155
+ optparser.add_option(
3156
+ '-c', '--colormap',
3157
+ type="choice", choices=('color', 'pink', 'gray', 'bw', 'print'),
3158
+ dest="theme", default="color",
3159
+ help="color map: color, pink, gray, bw, or print [default: %default]")
3160
+ optparser.add_option(
3161
+ '-s', '--strip',
3162
+ action="store_true",
3163
+ dest="strip", default=False,
3164
+ help="strip function parameters, template parameters, and const modifiers from demangled C++ function names")
3165
+ optparser.add_option(
3166
+ '-w', '--wrap',
3167
+ action="store_true",
3168
+ dest="wrap", default=False,
3169
+ help="wrap function names")
3170
+ optparser.add_option(
3171
+ '--show-samples',
3172
+ action="store_true",
3173
+ dest="show_samples", default=False,
3174
+ help="show function samples")
3175
+ # add option to create subtree or show paths
3176
+ optparser.add_option(
3177
+ '-z', '--root',
3178
+ type="string",
3179
+ dest="root", default="",
3180
+ help="prune call graph to show only descendants of specified root function")
3181
+ optparser.add_option(
3182
+ '-l', '--leaf',
3183
+ type="string",
3184
+ dest="leaf", default="",
3185
+ help="prune call graph to show only ancestors of specified leaf function")
3186
+ # add a new option to control skew of the colorization curve
3187
+ optparser.add_option(
3188
+ '--skew',
3189
+ type="float", dest="theme_skew", default=1.0,
3190
+ help="skew the colorization curve. Values < 1.0 give more variety to lower percentages. Values > 1.0 give less variety to lower percentages")
3191
+ (self.options, self.args) = optparser.parse_args(sys.argv[1:])
3192
+
3193
+ if len(self.args) > 1 and self.options.format != 'pstats':
3194
+ optparser.error('incorrect number of arguments')
3195
+
3196
+ try:
3197
+ self.theme = self.themes[self.options.theme]
3198
+ except KeyError:
3199
+ optparser.error('invalid colormap \'%s\'' % self.options.theme)
3200
+
3201
+ # set skew on the theme now that it has been picked.
3202
+ if self.options.theme_skew:
3203
+ self.theme.skew = self.options.theme_skew
3204
+
3205
+ totalMethod = self.options.totalMethod
3206
+
3207
+ try:
3208
+ Format = self.formats[self.options.format]
3209
+ except KeyError:
3210
+ optparser.error('invalid format \'%s\'' % self.options.format)
3211
+
3212
+ if Format.stdinInput:
3213
+ if not self.args:
3214
+ fp = sys.stdin
3215
+ else:
3216
+ fp = open(self.args[0], 'rt')
3217
+ parser = Format(fp)
3218
+ elif Format.multipleInput:
3219
+ if not self.args:
3220
+ optparser.error('at least a file must be specified for %s input' % self.options.format)
3221
+ parser = Format(*self.args)
3222
+ else:
3223
+ if len(self.args) != 1:
3224
+ optparser.error('exactly one file must be specified for %s input' % self.options.format)
3225
+ parser = Format(self.args[0])
3226
+
3227
+ self.profile = parser.parse()
3228
+
3229
+ if self.options.output is None:
3230
+ self.output = sys.stdout
3231
+ else:
3232
+ if PYTHON_3:
3233
+ self.output = open(self.options.output, 'wt', encoding='UTF-8')
3234
+ else:
3235
+ self.output = open(self.options.output, 'wt')
3236
+
3237
+ self.write_graph()
3238
+
3239
+ def write_graph(self):
3240
+ dot = DotWriter(self.output)
3241
+ dot.strip = self.options.strip
3242
+ dot.wrap = self.options.wrap
3243
+ if self.options.show_samples:
3244
+ dot.show_function_events.append(SAMPLES)
3245
+
3246
+ profile = self.profile
3247
+ profile.prune(self.options.node_thres/100.0, self.options.edge_thres/100.0)
3248
+
3249
+ if self.options.root:
3250
+ rootId = profile.getFunctionId(self.options.root)
3251
+ if not rootId:
3252
+ sys.stderr.write('root node ' + self.options.root + ' not found (might already be pruned : try -e0 -n0 flags)\n')
3253
+ sys.exit(1)
3254
+ profile.prune_root(rootId)
3255
+ if self.options.leaf:
3256
+ leafId = profile.getFunctionId(self.options.leaf)
3257
+ if not leafId:
3258
+ sys.stderr.write('leaf node ' + self.options.leaf + ' not found (maybe already pruned : try -e0 -n0 flags)\n')
3259
+ sys.exit(1)
3260
+ profile.prune_leaf(leafId)
3261
+
3262
+ dot.graph(profile, self.theme)
3263
+
3264
+
3265
+ if __name__ == '__main__':
3266
+ Main().main()