stackprof 0.2.2 → 0.2.3

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,3266 @@
1
+ #!/usr/bin/env python
2
+ #
3
+ # Copyright 2008-2009 Jose Fonseca
4
+ #
5
+ # This program is free software: you can redistribute it and/or modify it
6
+ # under the terms of the GNU Lesser General Public License as published
7
+ # by the Free Software Foundation, either version 3 of the License, or
8
+ # (at your option) any later version.
9
+ #
10
+ # This program is distributed in the hope that it will be useful,
11
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ # GNU Lesser General Public License for more details.
14
+ #
15
+ # You should have received a copy of the GNU Lesser General Public License
16
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
+ #
18
+
19
+ """Generate a dot graph from the output of several profilers."""
20
+
21
+ __author__ = "Jose Fonseca et al"
22
+
23
+
24
+ import sys
25
+ import math
26
+ import os.path
27
+ import re
28
+ import textwrap
29
+ import optparse
30
+ import xml.parsers.expat
31
+ import collections
32
+ import locale
33
+
34
+
35
+ # Python 2.x/3.x compatibility
36
+ if sys.version_info[0] >= 3:
37
+ PYTHON_3 = True
38
+ def compat_iteritems(x): return x.items() # No iteritems() in Python 3
39
+ def compat_itervalues(x): return x.values() # No itervalues() in Python 3
40
+ def compat_keys(x): return list(x.keys()) # keys() is a generator in Python 3
41
+ basestring = str # No class basestring in Python 3
42
+ unichr = chr # No unichr in Python 3
43
+ xrange = range # No xrange in Python 3
44
+ else:
45
+ PYTHON_3 = False
46
+ def compat_iteritems(x): return x.iteritems()
47
+ def compat_itervalues(x): return x.itervalues()
48
+ def compat_keys(x): return x.keys()
49
+
50
+
51
+ try:
52
+ # Debugging helper module
53
+ import debug
54
+ except ImportError:
55
+ pass
56
+
57
+
58
+ MULTIPLICATION_SIGN = unichr(0xd7)
59
+
60
+
61
+ def times(x):
62
+ return "%u%s" % (x, MULTIPLICATION_SIGN)
63
+
64
+ def percentage(p):
65
+ return "%.02f%%" % (p*100.0,)
66
+
67
+ def add(a, b):
68
+ return a + b
69
+
70
+ def equal(a, b):
71
+ if a == b:
72
+ return a
73
+ else:
74
+ return None
75
+
76
+ def fail(a, b):
77
+ assert False
78
+
79
+
80
+ tol = 2 ** -23
81
+
82
+ def ratio(numerator, denominator):
83
+ try:
84
+ ratio = float(numerator)/float(denominator)
85
+ except ZeroDivisionError:
86
+ # 0/0 is undefined, but 1.0 yields more useful results
87
+ return 1.0
88
+ if ratio < 0.0:
89
+ if ratio < -tol:
90
+ sys.stderr.write('warning: negative ratio (%s/%s)\n' % (numerator, denominator))
91
+ return 0.0
92
+ if ratio > 1.0:
93
+ if ratio > 1.0 + tol:
94
+ sys.stderr.write('warning: ratio greater than one (%s/%s)\n' % (numerator, denominator))
95
+ return 1.0
96
+ return ratio
97
+
98
+
99
+ class UndefinedEvent(Exception):
100
+ """Raised when attempting to get an event which is undefined."""
101
+
102
+ def __init__(self, event):
103
+ Exception.__init__(self)
104
+ self.event = event
105
+
106
+ def __str__(self):
107
+ return 'unspecified event %s' % self.event.name
108
+
109
+
110
+ class Event(object):
111
+ """Describe a kind of event, and its basic operations."""
112
+
113
+ def __init__(self, name, null, aggregator, formatter = str):
114
+ self.name = name
115
+ self._null = null
116
+ self._aggregator = aggregator
117
+ self._formatter = formatter
118
+
119
+ def __eq__(self, other):
120
+ return self is other
121
+
122
+ def __hash__(self):
123
+ return id(self)
124
+
125
+ def null(self):
126
+ return self._null
127
+
128
+ def aggregate(self, val1, val2):
129
+ """Aggregate two event values."""
130
+ assert val1 is not None
131
+ assert val2 is not None
132
+ return self._aggregator(val1, val2)
133
+
134
+ def format(self, val):
135
+ """Format an event value."""
136
+ assert val is not None
137
+ return self._formatter(val)
138
+
139
+
140
+ CALLS = Event("Calls", 0, add, times)
141
+ SAMPLES = Event("Samples", 0, add, times)
142
+ SAMPLES2 = Event("Samples", 0, add, times)
143
+
144
+ # Count of samples where a given function was either executing or on the stack.
145
+ # This is used to calculate the total time ratio according to the
146
+ # straightforward method described in Mike Dunlavey's answer to
147
+ # stackoverflow.com/questions/1777556/alternatives-to-gprof, item 4 (the myth
148
+ # "that recursion is a tricky confusing issue"), last edited 2012-08-30: it's
149
+ # just the ratio of TOTAL_SAMPLES over the number of samples in the profile.
150
+ #
151
+ # Used only when totalMethod == callstacks
152
+ TOTAL_SAMPLES = Event("Samples", 0, add, times)
153
+
154
+ TIME = Event("Time", 0.0, add, lambda x: '(' + str(x) + ')')
155
+ TIME_RATIO = Event("Time ratio", 0.0, add, lambda x: '(' + percentage(x) + ')')
156
+ TOTAL_TIME = Event("Total time", 0.0, fail)
157
+ TOTAL_TIME_RATIO = Event("Total time ratio", 0.0, fail, percentage)
158
+
159
+ totalMethod = 'callratios'
160
+
161
+
162
+ class Object(object):
163
+ """Base class for all objects in profile which can store events."""
164
+
165
+ def __init__(self, events=None):
166
+ if events is None:
167
+ self.events = {}
168
+ else:
169
+ self.events = events
170
+
171
+ def __hash__(self):
172
+ return id(self)
173
+
174
+ def __eq__(self, other):
175
+ return self is other
176
+
177
+ def __contains__(self, event):
178
+ return event in self.events
179
+
180
+ def __getitem__(self, event):
181
+ try:
182
+ return self.events[event]
183
+ except KeyError:
184
+ raise UndefinedEvent(event)
185
+
186
+ def __setitem__(self, event, value):
187
+ if value is None:
188
+ if event in self.events:
189
+ del self.events[event]
190
+ else:
191
+ self.events[event] = value
192
+
193
+
194
+ class Call(Object):
195
+ """A call between functions.
196
+
197
+ There should be at most one call object for every pair of functions.
198
+ """
199
+
200
+ def __init__(self, callee_id):
201
+ Object.__init__(self)
202
+ self.callee_id = callee_id
203
+ self.ratio = None
204
+ self.weight = None
205
+
206
+
207
+ class Function(Object):
208
+ """A function."""
209
+
210
+ def __init__(self, id, name):
211
+ Object.__init__(self)
212
+ self.id = id
213
+ self.name = name
214
+ self.module = None
215
+ self.process = None
216
+ self.calls = {}
217
+ self.called = None
218
+ self.weight = None
219
+ self.cycle = None
220
+
221
+ def add_call(self, call):
222
+ if call.callee_id in self.calls:
223
+ sys.stderr.write('warning: overwriting call from function %s to %s\n' % (str(self.id), str(call.callee_id)))
224
+ self.calls[call.callee_id] = call
225
+
226
+ def get_call(self, callee_id):
227
+ if not callee_id in self.calls:
228
+ call = Call(callee_id)
229
+ call[SAMPLES] = 0
230
+ call[SAMPLES2] = 0
231
+ call[CALLS] = 0
232
+ self.calls[callee_id] = call
233
+ return self.calls[callee_id]
234
+
235
+ _parenthesis_re = re.compile(r'\([^()]*\)')
236
+ _angles_re = re.compile(r'<[^<>]*>')
237
+ _const_re = re.compile(r'\s+const$')
238
+
239
+ def stripped_name(self):
240
+ """Remove extraneous information from C++ demangled function names."""
241
+
242
+ name = self.name
243
+
244
+ # Strip function parameters from name by recursively removing paired parenthesis
245
+ while True:
246
+ name, n = self._parenthesis_re.subn('', name)
247
+ if not n:
248
+ break
249
+
250
+ # Strip const qualifier
251
+ name = self._const_re.sub('', name)
252
+
253
+ # Strip template parameters from name by recursively removing paired angles
254
+ while True:
255
+ name, n = self._angles_re.subn('', name)
256
+ if not n:
257
+ break
258
+
259
+ return name
260
+
261
+ # TODO: write utility functions
262
+
263
+ def __repr__(self):
264
+ return self.name
265
+
266
+
267
+ class Cycle(Object):
268
+ """A cycle made from recursive function calls."""
269
+
270
+ def __init__(self):
271
+ Object.__init__(self)
272
+ # XXX: Do cycles need an id?
273
+ self.functions = set()
274
+
275
+ def add_function(self, function):
276
+ assert function not in self.functions
277
+ self.functions.add(function)
278
+ # XXX: Aggregate events?
279
+ if function.cycle is not None:
280
+ for other in function.cycle.functions:
281
+ if function not in self.functions:
282
+ self.add_function(other)
283
+ function.cycle = self
284
+
285
+
286
+ class Profile(Object):
287
+ """The whole profile."""
288
+
289
+ def __init__(self):
290
+ Object.__init__(self)
291
+ self.functions = {}
292
+ self.cycles = []
293
+
294
+ def add_function(self, function):
295
+ if function.id in self.functions:
296
+ sys.stderr.write('warning: overwriting function %s (id %s)\n' % (function.name, str(function.id)))
297
+ self.functions[function.id] = function
298
+
299
+ def add_cycle(self, cycle):
300
+ self.cycles.append(cycle)
301
+
302
+ def validate(self):
303
+ """Validate the edges."""
304
+
305
+ for function in compat_itervalues(self.functions):
306
+ for callee_id in compat_keys(function.calls):
307
+ assert function.calls[callee_id].callee_id == callee_id
308
+ if callee_id not in self.functions:
309
+ sys.stderr.write('warning: call to undefined function %s from function %s\n' % (str(callee_id), function.name))
310
+ del function.calls[callee_id]
311
+
312
+ def find_cycles(self):
313
+ """Find cycles using Tarjan's strongly connected components algorithm."""
314
+
315
+ # Apply the Tarjan's algorithm successively until all functions are visited
316
+ visited = set()
317
+ for function in compat_itervalues(self.functions):
318
+ if function not in visited:
319
+ self._tarjan(function, 0, [], {}, {}, visited)
320
+ cycles = []
321
+ for function in compat_itervalues(self.functions):
322
+ if function.cycle is not None and function.cycle not in cycles:
323
+ cycles.append(function.cycle)
324
+ self.cycles = cycles
325
+ if 0:
326
+ for cycle in cycles:
327
+ sys.stderr.write("Cycle:\n")
328
+ for member in cycle.functions:
329
+ sys.stderr.write("\tFunction %s\n" % member.name)
330
+
331
+ def prune_root(self, root):
332
+ visited = set()
333
+ frontier = set([root])
334
+ while len(frontier) > 0:
335
+ node = frontier.pop()
336
+ visited.add(node)
337
+ f = self.functions[node]
338
+ newNodes = f.calls.keys()
339
+ frontier = frontier.union(set(newNodes) - visited)
340
+ subtreeFunctions = {}
341
+ for n in visited:
342
+ subtreeFunctions[n] = self.functions[n]
343
+ self.functions = subtreeFunctions
344
+
345
+ def prune_leaf(self, leaf):
346
+ edgesUp = collections.defaultdict(set)
347
+ for f in self.functions.keys():
348
+ for n in self.functions[f].calls.keys():
349
+ edgesUp[n].add(f)
350
+ # build the tree up
351
+ visited = set()
352
+ frontier = set([leaf])
353
+ while len(frontier) > 0:
354
+ node = frontier.pop()
355
+ visited.add(node)
356
+ frontier = frontier.union(edgesUp[node] - visited)
357
+ downTree = set(self.functions.keys())
358
+ upTree = visited
359
+ path = downTree.intersection(upTree)
360
+ pathFunctions = {}
361
+ for n in path:
362
+ f = self.functions[n]
363
+ newCalls = {}
364
+ for c in f.calls.keys():
365
+ if c in path:
366
+ newCalls[c] = f.calls[c]
367
+ f.calls = newCalls
368
+ pathFunctions[n] = f
369
+ self.functions = pathFunctions
370
+
371
+
372
+ def getFunctionId(self, funcName):
373
+ for f in self.functions:
374
+ if self.functions[f].name == funcName:
375
+ return f
376
+ return False
377
+
378
+ def _tarjan(self, function, order, stack, orders, lowlinks, visited):
379
+ """Tarjan's strongly connected components algorithm.
380
+
381
+ See also:
382
+ - http://en.wikipedia.org/wiki/Tarjan's_strongly_connected_components_algorithm
383
+ """
384
+
385
+ visited.add(function)
386
+ orders[function] = order
387
+ lowlinks[function] = order
388
+ order += 1
389
+ pos = len(stack)
390
+ stack.append(function)
391
+ for call in compat_itervalues(function.calls):
392
+ callee = self.functions[call.callee_id]
393
+ # TODO: use a set to optimize lookup
394
+ if callee not in orders:
395
+ order = self._tarjan(callee, order, stack, orders, lowlinks, visited)
396
+ lowlinks[function] = min(lowlinks[function], lowlinks[callee])
397
+ elif callee in stack:
398
+ lowlinks[function] = min(lowlinks[function], orders[callee])
399
+ if lowlinks[function] == orders[function]:
400
+ # Strongly connected component found
401
+ members = stack[pos:]
402
+ del stack[pos:]
403
+ if len(members) > 1:
404
+ cycle = Cycle()
405
+ for member in members:
406
+ cycle.add_function(member)
407
+ return order
408
+
409
+ def call_ratios(self, event):
410
+ # Aggregate for incoming calls
411
+ cycle_totals = {}
412
+ for cycle in self.cycles:
413
+ cycle_totals[cycle] = 0.0
414
+ function_totals = {}
415
+ for function in compat_itervalues(self.functions):
416
+ function_totals[function] = 0.0
417
+
418
+ # Pass 1: function_total gets the sum of call[event] for all
419
+ # incoming arrows. Same for cycle_total for all arrows
420
+ # that are coming into the *cycle* but are not part of it.
421
+ for function in compat_itervalues(self.functions):
422
+ for call in compat_itervalues(function.calls):
423
+ if call.callee_id != function.id:
424
+ callee = self.functions[call.callee_id]
425
+ if event in call.events:
426
+ function_totals[callee] += call[event]
427
+ if callee.cycle is not None and callee.cycle is not function.cycle:
428
+ cycle_totals[callee.cycle] += call[event]
429
+ else:
430
+ sys.stderr.write("call_ratios: No data for " + function.name + " call to " + callee.name + "\n")
431
+
432
+ # Pass 2: Compute the ratios. Each call[event] is scaled by the
433
+ # function_total of the callee. Calls into cycles use the
434
+ # cycle_total, but not calls within cycles.
435
+ for function in compat_itervalues(self.functions):
436
+ for call in compat_itervalues(function.calls):
437
+ assert call.ratio is None
438
+ if call.callee_id != function.id:
439
+ callee = self.functions[call.callee_id]
440
+ if event in call.events:
441
+ if callee.cycle is not None and callee.cycle is not function.cycle:
442
+ total = cycle_totals[callee.cycle]
443
+ else:
444
+ total = function_totals[callee]
445
+ call.ratio = ratio(call[event], total)
446
+ else:
447
+ # Warnings here would only repeat those issued above.
448
+ call.ratio = 0.0
449
+
450
+ def integrate(self, outevent, inevent):
451
+ """Propagate function time ratio along the function calls.
452
+
453
+ Must be called after finding the cycles.
454
+
455
+ See also:
456
+ - http://citeseer.ist.psu.edu/graham82gprof.html
457
+ """
458
+
459
+ # Sanity checking
460
+ assert outevent not in self
461
+ for function in compat_itervalues(self.functions):
462
+ assert outevent not in function
463
+ assert inevent in function
464
+ for call in compat_itervalues(function.calls):
465
+ assert outevent not in call
466
+ if call.callee_id != function.id:
467
+ assert call.ratio is not None
468
+
469
+ # Aggregate the input for each cycle
470
+ for cycle in self.cycles:
471
+ total = inevent.null()
472
+ for function in compat_itervalues(self.functions):
473
+ total = inevent.aggregate(total, function[inevent])
474
+ self[inevent] = total
475
+
476
+ # Integrate along the edges
477
+ total = inevent.null()
478
+ for function in compat_itervalues(self.functions):
479
+ total = inevent.aggregate(total, function[inevent])
480
+ self._integrate_function(function, outevent, inevent)
481
+ self[outevent] = total
482
+
483
+ def _integrate_function(self, function, outevent, inevent):
484
+ if function.cycle is not None:
485
+ return self._integrate_cycle(function.cycle, outevent, inevent)
486
+ else:
487
+ if outevent not in function:
488
+ total = function[inevent]
489
+ for call in compat_itervalues(function.calls):
490
+ if call.callee_id != function.id:
491
+ total += self._integrate_call(call, outevent, inevent)
492
+ function[outevent] = total
493
+ return function[outevent]
494
+
495
+ def _integrate_call(self, call, outevent, inevent):
496
+ assert outevent not in call
497
+ assert call.ratio is not None
498
+ callee = self.functions[call.callee_id]
499
+ subtotal = call.ratio *self._integrate_function(callee, outevent, inevent)
500
+ call[outevent] = subtotal
501
+ return subtotal
502
+
503
+ def _integrate_cycle(self, cycle, outevent, inevent):
504
+ if outevent not in cycle:
505
+
506
+ # Compute the outevent for the whole cycle
507
+ total = inevent.null()
508
+ for member in cycle.functions:
509
+ subtotal = member[inevent]
510
+ for call in compat_itervalues(member.calls):
511
+ callee = self.functions[call.callee_id]
512
+ if callee.cycle is not cycle:
513
+ subtotal += self._integrate_call(call, outevent, inevent)
514
+ total += subtotal
515
+ cycle[outevent] = total
516
+
517
+ # Compute the time propagated to callers of this cycle
518
+ callees = {}
519
+ for function in compat_itervalues(self.functions):
520
+ if function.cycle is not cycle:
521
+ for call in compat_itervalues(function.calls):
522
+ callee = self.functions[call.callee_id]
523
+ if callee.cycle is cycle:
524
+ try:
525
+ callees[callee] += call.ratio
526
+ except KeyError:
527
+ callees[callee] = call.ratio
528
+
529
+ for member in cycle.functions:
530
+ member[outevent] = outevent.null()
531
+
532
+ for callee, call_ratio in compat_iteritems(callees):
533
+ ranks = {}
534
+ call_ratios = {}
535
+ partials = {}
536
+ self._rank_cycle_function(cycle, callee, 0, ranks)
537
+ self._call_ratios_cycle(cycle, callee, ranks, call_ratios, set())
538
+ partial = self._integrate_cycle_function(cycle, callee, call_ratio, partials, ranks, call_ratios, outevent, inevent)
539
+ assert partial == max(partials.values())
540
+ assert not total or abs(1.0 - partial/(call_ratio*total)) <= 0.001
541
+
542
+ return cycle[outevent]
543
+
544
+ def _rank_cycle_function(self, cycle, function, rank, ranks):
545
+ if function not in ranks or ranks[function] > rank:
546
+ ranks[function] = rank
547
+ for call in compat_itervalues(function.calls):
548
+ if call.callee_id != function.id:
549
+ callee = self.functions[call.callee_id]
550
+ if callee.cycle is cycle:
551
+ self._rank_cycle_function(cycle, callee, rank + 1, ranks)
552
+
553
+ def _call_ratios_cycle(self, cycle, function, ranks, call_ratios, visited):
554
+ if function not in visited:
555
+ visited.add(function)
556
+ for call in compat_itervalues(function.calls):
557
+ if call.callee_id != function.id:
558
+ callee = self.functions[call.callee_id]
559
+ if callee.cycle is cycle:
560
+ if ranks[callee] > ranks[function]:
561
+ call_ratios[callee] = call_ratios.get(callee, 0.0) + call.ratio
562
+ self._call_ratios_cycle(cycle, callee, ranks, call_ratios, visited)
563
+
564
+ def _integrate_cycle_function(self, cycle, function, partial_ratio, partials, ranks, call_ratios, outevent, inevent):
565
+ if function not in partials:
566
+ partial = partial_ratio*function[inevent]
567
+ for call in compat_itervalues(function.calls):
568
+ if call.callee_id != function.id:
569
+ callee = self.functions[call.callee_id]
570
+ if callee.cycle is not cycle:
571
+ assert outevent in call
572
+ partial += partial_ratio*call[outevent]
573
+ else:
574
+ if ranks[callee] > ranks[function]:
575
+ callee_partial = self._integrate_cycle_function(cycle, callee, partial_ratio, partials, ranks, call_ratios, outevent, inevent)
576
+ call_ratio = ratio(call.ratio, call_ratios[callee])
577
+ call_partial = call_ratio*callee_partial
578
+ try:
579
+ call[outevent] += call_partial
580
+ except UndefinedEvent:
581
+ call[outevent] = call_partial
582
+ partial += call_partial
583
+ partials[function] = partial
584
+ try:
585
+ function[outevent] += partial
586
+ except UndefinedEvent:
587
+ function[outevent] = partial
588
+ return partials[function]
589
+
590
+ def aggregate(self, event):
591
+ """Aggregate an event for the whole profile."""
592
+
593
+ total = event.null()
594
+ for function in compat_itervalues(self.functions):
595
+ try:
596
+ total = event.aggregate(total, function[event])
597
+ except UndefinedEvent:
598
+ return
599
+ self[event] = total
600
+
601
+ def ratio(self, outevent, inevent):
602
+ assert outevent not in self
603
+ assert inevent in self
604
+ for function in compat_itervalues(self.functions):
605
+ assert outevent not in function
606
+ assert inevent in function
607
+ function[outevent] = ratio(function[inevent], self[inevent])
608
+ for call in compat_itervalues(function.calls):
609
+ assert outevent not in call
610
+ if inevent in call:
611
+ call[outevent] = ratio(call[inevent], self[inevent])
612
+ self[outevent] = 1.0
613
+
614
+ def prune(self, node_thres, edge_thres):
615
+ """Prune the profile"""
616
+
617
+ # compute the prune ratios
618
+ for function in compat_itervalues(self.functions):
619
+ try:
620
+ function.weight = function[TOTAL_TIME_RATIO]
621
+ except UndefinedEvent:
622
+ pass
623
+
624
+ for call in compat_itervalues(function.calls):
625
+ callee = self.functions[call.callee_id]
626
+
627
+ if TOTAL_TIME_RATIO in call:
628
+ # handle exact cases first
629
+ call.weight = call[TOTAL_TIME_RATIO]
630
+ else:
631
+ try:
632
+ # make a safe estimate
633
+ call.weight = min(function[TOTAL_TIME_RATIO], callee[TOTAL_TIME_RATIO])
634
+ except UndefinedEvent:
635
+ pass
636
+
637
+ # prune the nodes
638
+ for function_id in compat_keys(self.functions):
639
+ function = self.functions[function_id]
640
+ if function.weight is not None:
641
+ if function.weight < node_thres:
642
+ del self.functions[function_id]
643
+
644
+ # prune the egdes
645
+ for function in compat_itervalues(self.functions):
646
+ for callee_id in compat_keys(function.calls):
647
+ call = function.calls[callee_id]
648
+ if callee_id not in self.functions or call.weight is not None and call.weight < edge_thres:
649
+ del function.calls[callee_id]
650
+
651
+ def dump(self):
652
+ for function in compat_itervalues(self.functions):
653
+ sys.stderr.write('Function %s:\n' % (function.name,))
654
+ self._dump_events(function.events)
655
+ for call in compat_itervalues(function.calls):
656
+ callee = self.functions[call.callee_id]
657
+ sys.stderr.write(' Call %s:\n' % (callee.name,))
658
+ self._dump_events(call.events)
659
+ for cycle in self.cycles:
660
+ sys.stderr.write('Cycle:\n')
661
+ self._dump_events(cycle.events)
662
+ for function in cycle.functions:
663
+ sys.stderr.write(' Function %s\n' % (function.name,))
664
+
665
+ def _dump_events(self, events):
666
+ for event, value in compat_iteritems(events):
667
+ sys.stderr.write(' %s: %s\n' % (event.name, event.format(value)))
668
+
669
+
670
+ class Struct:
671
+ """Masquerade a dictionary with a structure-like behavior."""
672
+
673
+ def __init__(self, attrs = None):
674
+ if attrs is None:
675
+ attrs = {}
676
+ self.__dict__['_attrs'] = attrs
677
+
678
+ def __getattr__(self, name):
679
+ try:
680
+ return self._attrs[name]
681
+ except KeyError:
682
+ raise AttributeError(name)
683
+
684
+ def __setattr__(self, name, value):
685
+ self._attrs[name] = value
686
+
687
+ def __str__(self):
688
+ return str(self._attrs)
689
+
690
+ def __repr__(self):
691
+ return repr(self._attrs)
692
+
693
+
694
+ class ParseError(Exception):
695
+ """Raised when parsing to signal mismatches."""
696
+
697
+ def __init__(self, msg, line):
698
+ self.msg = msg
699
+ # TODO: store more source line information
700
+ self.line = line
701
+
702
+ def __str__(self):
703
+ return '%s: %r' % (self.msg, self.line)
704
+
705
+
706
+ class Parser:
707
+ """Parser interface."""
708
+
709
+ stdinInput = True
710
+ multipleInput = False
711
+
712
+ def __init__(self):
713
+ pass
714
+
715
+ def parse(self):
716
+ raise NotImplementedError
717
+
718
+
719
+ class LineParser(Parser):
720
+ """Base class for parsers that read line-based formats."""
721
+
722
+ def __init__(self, stream):
723
+ Parser.__init__(self)
724
+ self._stream = stream
725
+ self.__line = None
726
+ self.__eof = False
727
+ self.line_no = 0
728
+
729
+ def readline(self):
730
+ line = self._stream.readline()
731
+ if not line:
732
+ self.__line = ''
733
+ self.__eof = True
734
+ else:
735
+ self.line_no += 1
736
+ line = line.rstrip('\r\n')
737
+ if not PYTHON_3:
738
+ encoding = self._stream.encoding
739
+ if encoding is None:
740
+ encoding = locale.getpreferredencoding()
741
+ line = line.decode(encoding)
742
+ self.__line = line
743
+
744
+ def lookahead(self):
745
+ assert self.__line is not None
746
+ return self.__line
747
+
748
+ def consume(self):
749
+ assert self.__line is not None
750
+ line = self.__line
751
+ self.readline()
752
+ return line
753
+
754
+ def eof(self):
755
+ assert self.__line is not None
756
+ return self.__eof
757
+
758
+
759
+ XML_ELEMENT_START, XML_ELEMENT_END, XML_CHARACTER_DATA, XML_EOF = range(4)
760
+
761
+
762
+ class XmlToken:
763
+
764
+ def __init__(self, type, name_or_data, attrs = None, line = None, column = None):
765
+ assert type in (XML_ELEMENT_START, XML_ELEMENT_END, XML_CHARACTER_DATA, XML_EOF)
766
+ self.type = type
767
+ self.name_or_data = name_or_data
768
+ self.attrs = attrs
769
+ self.line = line
770
+ self.column = column
771
+
772
+ def __str__(self):
773
+ if self.type == XML_ELEMENT_START:
774
+ return '<' + self.name_or_data + ' ...>'
775
+ if self.type == XML_ELEMENT_END:
776
+ return '</' + self.name_or_data + '>'
777
+ if self.type == XML_CHARACTER_DATA:
778
+ return self.name_or_data
779
+ if self.type == XML_EOF:
780
+ return 'end of file'
781
+ assert 0
782
+
783
+
784
+ class XmlTokenizer:
785
+ """Expat based XML tokenizer."""
786
+
787
+ def __init__(self, fp, skip_ws = True):
788
+ self.fp = fp
789
+ self.tokens = []
790
+ self.index = 0
791
+ self.final = False
792
+ self.skip_ws = skip_ws
793
+
794
+ self.character_pos = 0, 0
795
+ self.character_data = ''
796
+
797
+ self.parser = xml.parsers.expat.ParserCreate()
798
+ self.parser.StartElementHandler = self.handle_element_start
799
+ self.parser.EndElementHandler = self.handle_element_end
800
+ self.parser.CharacterDataHandler = self.handle_character_data
801
+
802
+ def handle_element_start(self, name, attributes):
803
+ self.finish_character_data()
804
+ line, column = self.pos()
805
+ token = XmlToken(XML_ELEMENT_START, name, attributes, line, column)
806
+ self.tokens.append(token)
807
+
808
+ def handle_element_end(self, name):
809
+ self.finish_character_data()
810
+ line, column = self.pos()
811
+ token = XmlToken(XML_ELEMENT_END, name, None, line, column)
812
+ self.tokens.append(token)
813
+
814
+ def handle_character_data(self, data):
815
+ if not self.character_data:
816
+ self.character_pos = self.pos()
817
+ self.character_data += data
818
+
819
+ def finish_character_data(self):
820
+ if self.character_data:
821
+ if not self.skip_ws or not self.character_data.isspace():
822
+ line, column = self.character_pos
823
+ token = XmlToken(XML_CHARACTER_DATA, self.character_data, None, line, column)
824
+ self.tokens.append(token)
825
+ self.character_data = ''
826
+
827
+ def next(self):
828
+ size = 16*1024
829
+ while self.index >= len(self.tokens) and not self.final:
830
+ self.tokens = []
831
+ self.index = 0
832
+ data = self.fp.read(size)
833
+ self.final = len(data) < size
834
+ try:
835
+ self.parser.Parse(data, self.final)
836
+ except xml.parsers.expat.ExpatError as e:
837
+ #if e.code == xml.parsers.expat.errors.XML_ERROR_NO_ELEMENTS:
838
+ if e.code == 3:
839
+ pass
840
+ else:
841
+ raise e
842
+ if self.index >= len(self.tokens):
843
+ line, column = self.pos()
844
+ token = XmlToken(XML_EOF, None, None, line, column)
845
+ else:
846
+ token = self.tokens[self.index]
847
+ self.index += 1
848
+ return token
849
+
850
+ def pos(self):
851
+ return self.parser.CurrentLineNumber, self.parser.CurrentColumnNumber
852
+
853
+
854
+ class XmlTokenMismatch(Exception):
855
+
856
+ def __init__(self, expected, found):
857
+ self.expected = expected
858
+ self.found = found
859
+
860
+ def __str__(self):
861
+ return '%u:%u: %s expected, %s found' % (self.found.line, self.found.column, str(self.expected), str(self.found))
862
+
863
+
864
+ class XmlParser(Parser):
865
+ """Base XML document parser."""
866
+
867
+ def __init__(self, fp):
868
+ Parser.__init__(self)
869
+ self.tokenizer = XmlTokenizer(fp)
870
+ self.consume()
871
+
872
+ def consume(self):
873
+ self.token = self.tokenizer.next()
874
+
875
+ def match_element_start(self, name):
876
+ return self.token.type == XML_ELEMENT_START and self.token.name_or_data == name
877
+
878
+ def match_element_end(self, name):
879
+ return self.token.type == XML_ELEMENT_END and self.token.name_or_data == name
880
+
881
+ def element_start(self, name):
882
+ while self.token.type == XML_CHARACTER_DATA:
883
+ self.consume()
884
+ if self.token.type != XML_ELEMENT_START:
885
+ raise XmlTokenMismatch(XmlToken(XML_ELEMENT_START, name), self.token)
886
+ if self.token.name_or_data != name:
887
+ raise XmlTokenMismatch(XmlToken(XML_ELEMENT_START, name), self.token)
888
+ attrs = self.token.attrs
889
+ self.consume()
890
+ return attrs
891
+
892
+ def element_end(self, name):
893
+ while self.token.type == XML_CHARACTER_DATA:
894
+ self.consume()
895
+ if self.token.type != XML_ELEMENT_END:
896
+ raise XmlTokenMismatch(XmlToken(XML_ELEMENT_END, name), self.token)
897
+ if self.token.name_or_data != name:
898
+ raise XmlTokenMismatch(XmlToken(XML_ELEMENT_END, name), self.token)
899
+ self.consume()
900
+
901
+ def character_data(self, strip = True):
902
+ data = ''
903
+ while self.token.type == XML_CHARACTER_DATA:
904
+ data += self.token.name_or_data
905
+ self.consume()
906
+ if strip:
907
+ data = data.strip()
908
+ return data
909
+
910
+
911
+ class GprofParser(Parser):
912
+ """Parser for GNU gprof output.
913
+
914
+ See also:
915
+ - Chapter "Interpreting gprof's Output" from the GNU gprof manual
916
+ http://sourceware.org/binutils/docs-2.18/gprof/Call-Graph.html#Call-Graph
917
+ - File "cg_print.c" from the GNU gprof source code
918
+ http://sourceware.org/cgi-bin/cvsweb.cgi/~checkout~/src/gprof/cg_print.c?rev=1.12&cvsroot=src
919
+ """
920
+
921
+ def __init__(self, fp):
922
+ Parser.__init__(self)
923
+ self.fp = fp
924
+ self.functions = {}
925
+ self.cycles = {}
926
+
927
+ def readline(self):
928
+ line = self.fp.readline()
929
+ if not line:
930
+ sys.stderr.write('error: unexpected end of file\n')
931
+ sys.exit(1)
932
+ line = line.rstrip('\r\n')
933
+ return line
934
+
935
+ _int_re = re.compile(r'^\d+$')
936
+ _float_re = re.compile(r'^\d+\.\d+$')
937
+
938
+ def translate(self, mo):
939
+ """Extract a structure from a match object, while translating the types in the process."""
940
+ attrs = {}
941
+ groupdict = mo.groupdict()
942
+ for name, value in compat_iteritems(groupdict):
943
+ if value is None:
944
+ value = None
945
+ elif self._int_re.match(value):
946
+ value = int(value)
947
+ elif self._float_re.match(value):
948
+ value = float(value)
949
+ attrs[name] = (value)
950
+ return Struct(attrs)
951
+
952
+ _cg_header_re = re.compile(
953
+ # original gprof header
954
+ r'^\s+called/total\s+parents\s*$|' +
955
+ r'^index\s+%time\s+self\s+descendents\s+called\+self\s+name\s+index\s*$|' +
956
+ r'^\s+called/total\s+children\s*$|' +
957
+ # GNU gprof header
958
+ r'^index\s+%\s+time\s+self\s+children\s+called\s+name\s*$'
959
+ )
960
+
961
+ _cg_ignore_re = re.compile(
962
+ # spontaneous
963
+ r'^\s+<spontaneous>\s*$|'
964
+ # internal calls (such as "mcount")
965
+ r'^.*\((\d+)\)$'
966
+ )
967
+
968
+ _cg_primary_re = re.compile(
969
+ r'^\[(?P<index>\d+)\]?' +
970
+ r'\s+(?P<percentage_time>\d+\.\d+)' +
971
+ r'\s+(?P<self>\d+\.\d+)' +
972
+ r'\s+(?P<descendants>\d+\.\d+)' +
973
+ r'\s+(?:(?P<called>\d+)(?:\+(?P<called_self>\d+))?)?' +
974
+ r'\s+(?P<name>\S.*?)' +
975
+ r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +
976
+ r'\s\[(\d+)\]$'
977
+ )
978
+
979
+ _cg_parent_re = re.compile(
980
+ r'^\s+(?P<self>\d+\.\d+)?' +
981
+ r'\s+(?P<descendants>\d+\.\d+)?' +
982
+ r'\s+(?P<called>\d+)(?:/(?P<called_total>\d+))?' +
983
+ r'\s+(?P<name>\S.*?)' +
984
+ r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +
985
+ r'\s\[(?P<index>\d+)\]$'
986
+ )
987
+
988
+ _cg_child_re = _cg_parent_re
989
+
990
+ _cg_cycle_header_re = re.compile(
991
+ r'^\[(?P<index>\d+)\]?' +
992
+ r'\s+(?P<percentage_time>\d+\.\d+)' +
993
+ r'\s+(?P<self>\d+\.\d+)' +
994
+ r'\s+(?P<descendants>\d+\.\d+)' +
995
+ r'\s+(?:(?P<called>\d+)(?:\+(?P<called_self>\d+))?)?' +
996
+ r'\s+<cycle\s(?P<cycle>\d+)\sas\sa\swhole>' +
997
+ r'\s\[(\d+)\]$'
998
+ )
999
+
1000
+ _cg_cycle_member_re = re.compile(
1001
+ r'^\s+(?P<self>\d+\.\d+)?' +
1002
+ r'\s+(?P<descendants>\d+\.\d+)?' +
1003
+ r'\s+(?P<called>\d+)(?:\+(?P<called_self>\d+))?' +
1004
+ r'\s+(?P<name>\S.*?)' +
1005
+ r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +
1006
+ r'\s\[(?P<index>\d+)\]$'
1007
+ )
1008
+
1009
+ _cg_sep_re = re.compile(r'^--+$')
1010
+
1011
+ def parse_function_entry(self, lines):
1012
+ parents = []
1013
+ children = []
1014
+
1015
+ while True:
1016
+ if not lines:
1017
+ sys.stderr.write('warning: unexpected end of entry\n')
1018
+ line = lines.pop(0)
1019
+ if line.startswith('['):
1020
+ break
1021
+
1022
+ # read function parent line
1023
+ mo = self._cg_parent_re.match(line)
1024
+ if not mo:
1025
+ if self._cg_ignore_re.match(line):
1026
+ continue
1027
+ sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line)
1028
+ else:
1029
+ parent = self.translate(mo)
1030
+ parents.append(parent)
1031
+
1032
+ # read primary line
1033
+ mo = self._cg_primary_re.match(line)
1034
+ if not mo:
1035
+ sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line)
1036
+ return
1037
+ else:
1038
+ function = self.translate(mo)
1039
+
1040
+ while lines:
1041
+ line = lines.pop(0)
1042
+
1043
+ # read function subroutine line
1044
+ mo = self._cg_child_re.match(line)
1045
+ if not mo:
1046
+ if self._cg_ignore_re.match(line):
1047
+ continue
1048
+ sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line)
1049
+ else:
1050
+ child = self.translate(mo)
1051
+ children.append(child)
1052
+
1053
+ function.parents = parents
1054
+ function.children = children
1055
+
1056
+ self.functions[function.index] = function
1057
+
1058
+ def parse_cycle_entry(self, lines):
1059
+
1060
+ # read cycle header line
1061
+ line = lines[0]
1062
+ mo = self._cg_cycle_header_re.match(line)
1063
+ if not mo:
1064
+ sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line)
1065
+ return
1066
+ cycle = self.translate(mo)
1067
+
1068
+ # read cycle member lines
1069
+ cycle.functions = []
1070
+ for line in lines[1:]:
1071
+ mo = self._cg_cycle_member_re.match(line)
1072
+ if not mo:
1073
+ sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line)
1074
+ continue
1075
+ call = self.translate(mo)
1076
+ cycle.functions.append(call)
1077
+
1078
+ self.cycles[cycle.cycle] = cycle
1079
+
1080
+ def parse_cg_entry(self, lines):
1081
+ if lines[0].startswith("["):
1082
+ self.parse_cycle_entry(lines)
1083
+ else:
1084
+ self.parse_function_entry(lines)
1085
+
1086
+ def parse_cg(self):
1087
+ """Parse the call graph."""
1088
+
1089
+ # skip call graph header
1090
+ while not self._cg_header_re.match(self.readline()):
1091
+ pass
1092
+ line = self.readline()
1093
+ while self._cg_header_re.match(line):
1094
+ line = self.readline()
1095
+
1096
+ # process call graph entries
1097
+ entry_lines = []
1098
+ while line != '\014': # form feed
1099
+ if line and not line.isspace():
1100
+ if self._cg_sep_re.match(line):
1101
+ self.parse_cg_entry(entry_lines)
1102
+ entry_lines = []
1103
+ else:
1104
+ entry_lines.append(line)
1105
+ line = self.readline()
1106
+
1107
+ def parse(self):
1108
+ self.parse_cg()
1109
+ self.fp.close()
1110
+
1111
+ profile = Profile()
1112
+ profile[TIME] = 0.0
1113
+
1114
+ cycles = {}
1115
+ for index in self.cycles:
1116
+ cycles[index] = Cycle()
1117
+
1118
+ for entry in compat_itervalues(self.functions):
1119
+ # populate the function
1120
+ function = Function(entry.index, entry.name)
1121
+ function[TIME] = entry.self
1122
+ if entry.called is not None:
1123
+ function.called = entry.called
1124
+ if entry.called_self is not None:
1125
+ call = Call(entry.index)
1126
+ call[CALLS] = entry.called_self
1127
+ function.called += entry.called_self
1128
+
1129
+ # populate the function calls
1130
+ for child in entry.children:
1131
+ call = Call(child.index)
1132
+
1133
+ assert child.called is not None
1134
+ call[CALLS] = child.called
1135
+
1136
+ if child.index not in self.functions:
1137
+ # NOTE: functions that were never called but were discovered by gprof's
1138
+ # static call graph analysis dont have a call graph entry so we need
1139
+ # to add them here
1140
+ missing = Function(child.index, child.name)
1141
+ function[TIME] = 0.0
1142
+ function.called = 0
1143
+ profile.add_function(missing)
1144
+
1145
+ function.add_call(call)
1146
+
1147
+ profile.add_function(function)
1148
+
1149
+ if entry.cycle is not None:
1150
+ try:
1151
+ cycle = cycles[entry.cycle]
1152
+ except KeyError:
1153
+ sys.stderr.write('warning: <cycle %u as a whole> entry missing\n' % entry.cycle)
1154
+ cycle = Cycle()
1155
+ cycles[entry.cycle] = cycle
1156
+ cycle.add_function(function)
1157
+
1158
+ profile[TIME] = profile[TIME] + function[TIME]
1159
+
1160
+ for cycle in compat_itervalues(cycles):
1161
+ profile.add_cycle(cycle)
1162
+
1163
+ # Compute derived events
1164
+ profile.validate()
1165
+ profile.ratio(TIME_RATIO, TIME)
1166
+ profile.call_ratios(CALLS)
1167
+ profile.integrate(TOTAL_TIME, TIME)
1168
+ profile.ratio(TOTAL_TIME_RATIO, TOTAL_TIME)
1169
+
1170
+ return profile
1171
+
1172
+
1173
+ # Clone&hack of GprofParser for VTune Amplifier XE 2013 gprof-cc output.
1174
+ # Tested only with AXE 2013 for Windows.
1175
+ # - Use total times as reported by AXE.
1176
+ # - In the absence of call counts, call ratios are faked from the relative
1177
+ # proportions of total time. This affects only the weighting of the calls.
1178
+ # - Different header, separator, and end marker.
1179
+ # - Extra whitespace after function names.
1180
+ # - You get a full entry for <spontaneous>, which does not have parents.
1181
+ # - Cycles do have parents. These are saved but unused (as they are
1182
+ # for functions).
1183
+ # - Disambiguated "unrecognized call graph entry" error messages.
1184
+ # Notes:
1185
+ # - Total time of functions as reported by AXE passes the val3 test.
1186
+ # - CPU Time:Children in the input is sometimes a negative number. This
1187
+ # value goes to the variable descendants, which is unused.
1188
+ # - The format of gprof-cc reports is unaffected by the use of
1189
+ # -knob enable-call-counts=true (no call counts, ever), or
1190
+ # -show-as=samples (results are quoted in seconds regardless).
1191
+ class AXEParser(Parser):
1192
+ "Parser for VTune Amplifier XE 2013 gprof-cc report output."
1193
+
1194
+ def __init__(self, fp):
1195
+ Parser.__init__(self)
1196
+ self.fp = fp
1197
+ self.functions = {}
1198
+ self.cycles = {}
1199
+
1200
+ def readline(self):
1201
+ line = self.fp.readline()
1202
+ if not line:
1203
+ sys.stderr.write('error: unexpected end of file\n')
1204
+ sys.exit(1)
1205
+ line = line.rstrip('\r\n')
1206
+ return line
1207
+
1208
+ _int_re = re.compile(r'^\d+$')
1209
+ _float_re = re.compile(r'^\d+\.\d+$')
1210
+
1211
+ def translate(self, mo):
1212
+ """Extract a structure from a match object, while translating the types in the process."""
1213
+ attrs = {}
1214
+ groupdict = mo.groupdict()
1215
+ for name, value in compat_iteritems(groupdict):
1216
+ if value is None:
1217
+ value = None
1218
+ elif self._int_re.match(value):
1219
+ value = int(value)
1220
+ elif self._float_re.match(value):
1221
+ value = float(value)
1222
+ attrs[name] = (value)
1223
+ return Struct(attrs)
1224
+
1225
+ _cg_header_re = re.compile(
1226
+ '^Index |'
1227
+ '^-----+ '
1228
+ )
1229
+
1230
+ _cg_footer_re = re.compile('^Index\s+Function\s*$')
1231
+
1232
+ _cg_primary_re = re.compile(
1233
+ r'^\[(?P<index>\d+)\]?' +
1234
+ r'\s+(?P<percentage_time>\d+\.\d+)' +
1235
+ r'\s+(?P<self>\d+\.\d+)' +
1236
+ r'\s+(?P<descendants>\d+\.\d+)' +
1237
+ r'\s+(?P<name>\S.*?)' +
1238
+ r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +
1239
+ r'\s+\[(\d+)\]$'
1240
+ )
1241
+
1242
+ _cg_parent_re = re.compile(
1243
+ r'^\s+(?P<self>\d+\.\d+)?' +
1244
+ r'\s+(?P<descendants>\d+\.\d+)?' +
1245
+ r'\s+(?P<name>\S.*?)' +
1246
+ r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +
1247
+ r'\s+\[(?P<index>\d+)\]$'
1248
+ )
1249
+
1250
+ _cg_child_re = _cg_parent_re
1251
+
1252
+ _cg_cycle_header_re = re.compile(
1253
+ r'^\[(?P<index>\d+)\]?' +
1254
+ r'\s+(?P<percentage_time>\d+\.\d+)' +
1255
+ r'\s+(?P<self>\d+\.\d+)' +
1256
+ r'\s+(?P<descendants>\d+\.\d+)' +
1257
+ r'\s+<cycle\s(?P<cycle>\d+)\sas\sa\swhole>' +
1258
+ r'\s+\[(\d+)\]$'
1259
+ )
1260
+
1261
+ _cg_cycle_member_re = re.compile(
1262
+ r'^\s+(?P<self>\d+\.\d+)?' +
1263
+ r'\s+(?P<descendants>\d+\.\d+)?' +
1264
+ r'\s+(?P<name>\S.*?)' +
1265
+ r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +
1266
+ r'\s+\[(?P<index>\d+)\]$'
1267
+ )
1268
+
1269
+ def parse_function_entry(self, lines):
1270
+ parents = []
1271
+ children = []
1272
+
1273
+ while True:
1274
+ if not lines:
1275
+ sys.stderr.write('warning: unexpected end of entry\n')
1276
+ return
1277
+ line = lines.pop(0)
1278
+ if line.startswith('['):
1279
+ break
1280
+
1281
+ # read function parent line
1282
+ mo = self._cg_parent_re.match(line)
1283
+ if not mo:
1284
+ sys.stderr.write('warning: unrecognized call graph entry (1): %r\n' % line)
1285
+ else:
1286
+ parent = self.translate(mo)
1287
+ if parent.name != '<spontaneous>':
1288
+ parents.append(parent)
1289
+
1290
+ # read primary line
1291
+ mo = self._cg_primary_re.match(line)
1292
+ if not mo:
1293
+ sys.stderr.write('warning: unrecognized call graph entry (2): %r\n' % line)
1294
+ return
1295
+ else:
1296
+ function = self.translate(mo)
1297
+
1298
+ while lines:
1299
+ line = lines.pop(0)
1300
+
1301
+ # read function subroutine line
1302
+ mo = self._cg_child_re.match(line)
1303
+ if not mo:
1304
+ sys.stderr.write('warning: unrecognized call graph entry (3): %r\n' % line)
1305
+ else:
1306
+ child = self.translate(mo)
1307
+ if child.name != '<spontaneous>':
1308
+ children.append(child)
1309
+
1310
+ if function.name != '<spontaneous>':
1311
+ function.parents = parents
1312
+ function.children = children
1313
+
1314
+ self.functions[function.index] = function
1315
+
1316
+ def parse_cycle_entry(self, lines):
1317
+
1318
+ # Process the parents that were not there in gprof format.
1319
+ parents = []
1320
+ while True:
1321
+ if not lines:
1322
+ sys.stderr.write('warning: unexpected end of cycle entry\n')
1323
+ return
1324
+ line = lines.pop(0)
1325
+ if line.startswith('['):
1326
+ break
1327
+ mo = self._cg_parent_re.match(line)
1328
+ if not mo:
1329
+ sys.stderr.write('warning: unrecognized call graph entry (6): %r\n' % line)
1330
+ else:
1331
+ parent = self.translate(mo)
1332
+ if parent.name != '<spontaneous>':
1333
+ parents.append(parent)
1334
+
1335
+ # read cycle header line
1336
+ mo = self._cg_cycle_header_re.match(line)
1337
+ if not mo:
1338
+ sys.stderr.write('warning: unrecognized call graph entry (4): %r\n' % line)
1339
+ return
1340
+ cycle = self.translate(mo)
1341
+
1342
+ # read cycle member lines
1343
+ cycle.functions = []
1344
+ for line in lines[1:]:
1345
+ mo = self._cg_cycle_member_re.match(line)
1346
+ if not mo:
1347
+ sys.stderr.write('warning: unrecognized call graph entry (5): %r\n' % line)
1348
+ continue
1349
+ call = self.translate(mo)
1350
+ cycle.functions.append(call)
1351
+
1352
+ cycle.parents = parents
1353
+ self.cycles[cycle.cycle] = cycle
1354
+
1355
+ def parse_cg_entry(self, lines):
1356
+ if any("as a whole" in linelooper for linelooper in lines):
1357
+ self.parse_cycle_entry(lines)
1358
+ else:
1359
+ self.parse_function_entry(lines)
1360
+
1361
+ def parse_cg(self):
1362
+ """Parse the call graph."""
1363
+
1364
+ # skip call graph header
1365
+ line = self.readline()
1366
+ while self._cg_header_re.match(line):
1367
+ line = self.readline()
1368
+
1369
+ # process call graph entries
1370
+ entry_lines = []
1371
+ # An EOF in readline terminates the program without returning.
1372
+ while not self._cg_footer_re.match(line):
1373
+ if line.isspace():
1374
+ self.parse_cg_entry(entry_lines)
1375
+ entry_lines = []
1376
+ else:
1377
+ entry_lines.append(line)
1378
+ line = self.readline()
1379
+
1380
+ def parse(self):
1381
+ sys.stderr.write('warning: for axe format, edge weights are unreliable estimates derived from\nfunction total times.\n')
1382
+ self.parse_cg()
1383
+ self.fp.close()
1384
+
1385
+ profile = Profile()
1386
+ profile[TIME] = 0.0
1387
+
1388
+ cycles = {}
1389
+ for index in self.cycles:
1390
+ cycles[index] = Cycle()
1391
+
1392
+ for entry in compat_itervalues(self.functions):
1393
+ # populate the function
1394
+ function = Function(entry.index, entry.name)
1395
+ function[TIME] = entry.self
1396
+ function[TOTAL_TIME_RATIO] = entry.percentage_time / 100.0
1397
+
1398
+ # populate the function calls
1399
+ for child in entry.children:
1400
+ call = Call(child.index)
1401
+ # The following bogus value affects only the weighting of
1402
+ # the calls.
1403
+ call[TOTAL_TIME_RATIO] = function[TOTAL_TIME_RATIO]
1404
+
1405
+ if child.index not in self.functions:
1406
+ # NOTE: functions that were never called but were discovered by gprof's
1407
+ # static call graph analysis dont have a call graph entry so we need
1408
+ # to add them here
1409
+ # FIXME: Is this applicable?
1410
+ missing = Function(child.index, child.name)
1411
+ function[TIME] = 0.0
1412
+ profile.add_function(missing)
1413
+
1414
+ function.add_call(call)
1415
+
1416
+ profile.add_function(function)
1417
+
1418
+ if entry.cycle is not None:
1419
+ try:
1420
+ cycle = cycles[entry.cycle]
1421
+ except KeyError:
1422
+ sys.stderr.write('warning: <cycle %u as a whole> entry missing\n' % entry.cycle)
1423
+ cycle = Cycle()
1424
+ cycles[entry.cycle] = cycle
1425
+ cycle.add_function(function)
1426
+
1427
+ profile[TIME] = profile[TIME] + function[TIME]
1428
+
1429
+ for cycle in compat_itervalues(cycles):
1430
+ profile.add_cycle(cycle)
1431
+
1432
+ # Compute derived events.
1433
+ profile.validate()
1434
+ profile.ratio(TIME_RATIO, TIME)
1435
+ # Lacking call counts, fake call ratios based on total times.
1436
+ profile.call_ratios(TOTAL_TIME_RATIO)
1437
+ # The TOTAL_TIME_RATIO of functions is already set. Propagate that
1438
+ # total time to the calls. (TOTAL_TIME is neither set nor used.)
1439
+ for function in compat_itervalues(profile.functions):
1440
+ for call in compat_itervalues(function.calls):
1441
+ if call.ratio is not None:
1442
+ callee = profile.functions[call.callee_id]
1443
+ call[TOTAL_TIME_RATIO] = call.ratio * callee[TOTAL_TIME_RATIO];
1444
+
1445
+ return profile
1446
+
1447
+
1448
+ class CallgrindParser(LineParser):
1449
+ """Parser for valgrind's callgrind tool.
1450
+
1451
+ See also:
1452
+ - http://valgrind.org/docs/manual/cl-format.html
1453
+ """
1454
+
1455
+ _call_re = re.compile('^calls=\s*(\d+)\s+((\d+|\+\d+|-\d+|\*)\s+)+$')
1456
+
1457
+ def __init__(self, infile):
1458
+ LineParser.__init__(self, infile)
1459
+
1460
+ # Textual positions
1461
+ self.position_ids = {}
1462
+ self.positions = {}
1463
+
1464
+ # Numeric positions
1465
+ self.num_positions = 1
1466
+ self.cost_positions = ['line']
1467
+ self.last_positions = [0]
1468
+
1469
+ # Events
1470
+ self.num_events = 0
1471
+ self.cost_events = []
1472
+
1473
+ self.profile = Profile()
1474
+ self.profile[SAMPLES] = 0
1475
+
1476
+ def parse(self):
1477
+ # read lookahead
1478
+ self.readline()
1479
+
1480
+ self.parse_key('version')
1481
+ self.parse_key('creator')
1482
+ while self.parse_part():
1483
+ pass
1484
+ if not self.eof():
1485
+ sys.stderr.write('warning: line %u: unexpected line\n' % self.line_no)
1486
+ sys.stderr.write('%s\n' % self.lookahead())
1487
+
1488
+ # compute derived data
1489
+ self.profile.validate()
1490
+ self.profile.find_cycles()
1491
+ self.profile.ratio(TIME_RATIO, SAMPLES)
1492
+ self.profile.call_ratios(CALLS)
1493
+ self.profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
1494
+
1495
+ return self.profile
1496
+
1497
+ def parse_part(self):
1498
+ if not self.parse_header_line():
1499
+ return False
1500
+ while self.parse_header_line():
1501
+ pass
1502
+ if not self.parse_body_line():
1503
+ return False
1504
+ while self.parse_body_line():
1505
+ pass
1506
+ return True
1507
+
1508
+ def parse_header_line(self):
1509
+ return \
1510
+ self.parse_empty() or \
1511
+ self.parse_comment() or \
1512
+ self.parse_part_detail() or \
1513
+ self.parse_description() or \
1514
+ self.parse_event_specification() or \
1515
+ self.parse_cost_line_def() or \
1516
+ self.parse_cost_summary()
1517
+
1518
+ _detail_keys = set(('cmd', 'pid', 'thread', 'part'))
1519
+
1520
+ def parse_part_detail(self):
1521
+ return self.parse_keys(self._detail_keys)
1522
+
1523
+ def parse_description(self):
1524
+ return self.parse_key('desc') is not None
1525
+
1526
+ def parse_event_specification(self):
1527
+ event = self.parse_key('event')
1528
+ if event is None:
1529
+ return False
1530
+ return True
1531
+
1532
+ def parse_cost_line_def(self):
1533
+ pair = self.parse_keys(('events', 'positions'))
1534
+ if pair is None:
1535
+ return False
1536
+ key, value = pair
1537
+ items = value.split()
1538
+ if key == 'events':
1539
+ self.num_events = len(items)
1540
+ self.cost_events = items
1541
+ if key == 'positions':
1542
+ self.num_positions = len(items)
1543
+ self.cost_positions = items
1544
+ self.last_positions = [0]*self.num_positions
1545
+ return True
1546
+
1547
+ def parse_cost_summary(self):
1548
+ pair = self.parse_keys(('summary', 'totals'))
1549
+ if pair is None:
1550
+ return False
1551
+ return True
1552
+
1553
+ def parse_body_line(self):
1554
+ return \
1555
+ self.parse_empty() or \
1556
+ self.parse_comment() or \
1557
+ self.parse_cost_line() or \
1558
+ self.parse_position_spec() or \
1559
+ self.parse_association_spec()
1560
+
1561
+ __subpos_re = r'(0x[0-9a-fA-F]+|\d+|\+\d+|-\d+|\*)'
1562
+ _cost_re = re.compile(r'^' +
1563
+ __subpos_re + r'( +' + __subpos_re + r')*' +
1564
+ r'( +\d+)*' +
1565
+ '$')
1566
+
1567
+ def parse_cost_line(self, calls=None):
1568
+ line = self.lookahead().rstrip()
1569
+ mo = self._cost_re.match(line)
1570
+ if not mo:
1571
+ return False
1572
+
1573
+ function = self.get_function()
1574
+
1575
+ if calls is None:
1576
+ # Unlike other aspects, call object (cob) is relative not to the
1577
+ # last call object, but to the caller's object (ob), so try to
1578
+ # update it when processing a functions cost line
1579
+ try:
1580
+ self.positions['cob'] = self.positions['ob']
1581
+ except KeyError:
1582
+ pass
1583
+
1584
+ values = line.split()
1585
+ assert len(values) <= self.num_positions + self.num_events
1586
+
1587
+ positions = values[0 : self.num_positions]
1588
+ events = values[self.num_positions : ]
1589
+ events += ['0']*(self.num_events - len(events))
1590
+
1591
+ for i in range(self.num_positions):
1592
+ position = positions[i]
1593
+ if position == '*':
1594
+ position = self.last_positions[i]
1595
+ elif position[0] in '-+':
1596
+ position = self.last_positions[i] + int(position)
1597
+ elif position.startswith('0x'):
1598
+ position = int(position, 16)
1599
+ else:
1600
+ position = int(position)
1601
+ self.last_positions[i] = position
1602
+
1603
+ events = [float(event) for event in events]
1604
+
1605
+ if calls is None:
1606
+ function[SAMPLES] += events[0]
1607
+ self.profile[SAMPLES] += events[0]
1608
+ else:
1609
+ callee = self.get_callee()
1610
+ callee.called += calls
1611
+
1612
+ try:
1613
+ call = function.calls[callee.id]
1614
+ except KeyError:
1615
+ call = Call(callee.id)
1616
+ call[CALLS] = calls
1617
+ call[SAMPLES] = events[0]
1618
+ function.add_call(call)
1619
+ else:
1620
+ call[CALLS] += calls
1621
+ call[SAMPLES] += events[0]
1622
+
1623
+ self.consume()
1624
+ return True
1625
+
1626
+ def parse_association_spec(self):
1627
+ line = self.lookahead()
1628
+ if not line.startswith('calls='):
1629
+ return False
1630
+
1631
+ _, values = line.split('=', 1)
1632
+ values = values.strip().split()
1633
+ calls = int(values[0])
1634
+ call_position = values[1:]
1635
+ self.consume()
1636
+
1637
+ self.parse_cost_line(calls)
1638
+
1639
+ return True
1640
+
1641
+ _position_re = re.compile('^(?P<position>[cj]?(?:ob|fl|fi|fe|fn))=\s*(?:\((?P<id>\d+)\))?(?:\s*(?P<name>.+))?')
1642
+
1643
+ _position_table_map = {
1644
+ 'ob': 'ob',
1645
+ 'fl': 'fl',
1646
+ 'fi': 'fl',
1647
+ 'fe': 'fl',
1648
+ 'fn': 'fn',
1649
+ 'cob': 'ob',
1650
+ 'cfl': 'fl',
1651
+ 'cfi': 'fl',
1652
+ 'cfe': 'fl',
1653
+ 'cfn': 'fn',
1654
+ 'jfi': 'fl',
1655
+ }
1656
+
1657
+ _position_map = {
1658
+ 'ob': 'ob',
1659
+ 'fl': 'fl',
1660
+ 'fi': 'fl',
1661
+ 'fe': 'fl',
1662
+ 'fn': 'fn',
1663
+ 'cob': 'cob',
1664
+ 'cfl': 'cfl',
1665
+ 'cfi': 'cfl',
1666
+ 'cfe': 'cfl',
1667
+ 'cfn': 'cfn',
1668
+ 'jfi': 'jfi',
1669
+ }
1670
+
1671
+ def parse_position_spec(self):
1672
+ line = self.lookahead()
1673
+
1674
+ if line.startswith('jump=') or line.startswith('jcnd='):
1675
+ self.consume()
1676
+ return True
1677
+
1678
+ mo = self._position_re.match(line)
1679
+ if not mo:
1680
+ return False
1681
+
1682
+ position, id, name = mo.groups()
1683
+ if id:
1684
+ table = self._position_table_map[position]
1685
+ if name:
1686
+ self.position_ids[(table, id)] = name
1687
+ else:
1688
+ name = self.position_ids.get((table, id), '')
1689
+ self.positions[self._position_map[position]] = name
1690
+
1691
+ self.consume()
1692
+ return True
1693
+
1694
+ def parse_empty(self):
1695
+ if self.eof():
1696
+ return False
1697
+ line = self.lookahead()
1698
+ if line.strip():
1699
+ return False
1700
+ self.consume()
1701
+ return True
1702
+
1703
+ def parse_comment(self):
1704
+ line = self.lookahead()
1705
+ if not line.startswith('#'):
1706
+ return False
1707
+ self.consume()
1708
+ return True
1709
+
1710
+ _key_re = re.compile(r'^(\w+):')
1711
+
1712
+ def parse_key(self, key):
1713
+ pair = self.parse_keys((key,))
1714
+ if not pair:
1715
+ return None
1716
+ key, value = pair
1717
+ return value
1718
+ line = self.lookahead()
1719
+ mo = self._key_re.match(line)
1720
+ if not mo:
1721
+ return None
1722
+ key, value = line.split(':', 1)
1723
+ if key not in keys:
1724
+ return None
1725
+ value = value.strip()
1726
+ self.consume()
1727
+ return key, value
1728
+
1729
+ def parse_keys(self, keys):
1730
+ line = self.lookahead()
1731
+ mo = self._key_re.match(line)
1732
+ if not mo:
1733
+ return None
1734
+ key, value = line.split(':', 1)
1735
+ if key not in keys:
1736
+ return None
1737
+ value = value.strip()
1738
+ self.consume()
1739
+ return key, value
1740
+
1741
+ def make_function(self, module, filename, name):
1742
+ # FIXME: module and filename are not being tracked reliably
1743
+ #id = '|'.join((module, filename, name))
1744
+ id = name
1745
+ try:
1746
+ function = self.profile.functions[id]
1747
+ except KeyError:
1748
+ function = Function(id, name)
1749
+ if module:
1750
+ function.module = os.path.basename(module)
1751
+ function[SAMPLES] = 0
1752
+ function.called = 0
1753
+ self.profile.add_function(function)
1754
+ return function
1755
+
1756
+ def get_function(self):
1757
+ module = self.positions.get('ob', '')
1758
+ filename = self.positions.get('fl', '')
1759
+ function = self.positions.get('fn', '')
1760
+ return self.make_function(module, filename, function)
1761
+
1762
+ def get_callee(self):
1763
+ module = self.positions.get('cob', '')
1764
+ filename = self.positions.get('cfi', '')
1765
+ function = self.positions.get('cfn', '')
1766
+ return self.make_function(module, filename, function)
1767
+
1768
+
1769
+ class PerfParser(LineParser):
1770
+ """Parser for linux perf callgraph output.
1771
+
1772
+ It expects output generated with
1773
+
1774
+ perf record -g
1775
+ perf script | gprof2dot.py --format=perf
1776
+ """
1777
+
1778
+ def __init__(self, infile):
1779
+ LineParser.__init__(self, infile)
1780
+ self.profile = Profile()
1781
+
1782
+ def readline(self):
1783
+ # Override LineParser.readline to ignore comment lines
1784
+ while True:
1785
+ LineParser.readline(self)
1786
+ if self.eof() or not self.lookahead().startswith('#'):
1787
+ break
1788
+
1789
+ def parse(self):
1790
+ # read lookahead
1791
+ self.readline()
1792
+
1793
+ profile = self.profile
1794
+ profile[SAMPLES] = 0
1795
+ while not self.eof():
1796
+ self.parse_event()
1797
+
1798
+ # compute derived data
1799
+ profile.validate()
1800
+ profile.find_cycles()
1801
+ profile.ratio(TIME_RATIO, SAMPLES)
1802
+ profile.call_ratios(SAMPLES2)
1803
+ if totalMethod == "callratios":
1804
+ # Heuristic approach. TOTAL_SAMPLES is unused.
1805
+ profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
1806
+ elif totalMethod == "callstacks":
1807
+ # Use the actual call chains for functions.
1808
+ profile[TOTAL_SAMPLES] = profile[SAMPLES]
1809
+ profile.ratio(TOTAL_TIME_RATIO, TOTAL_SAMPLES)
1810
+ # Then propagate that total time to the calls.
1811
+ for function in compat_itervalues(profile.functions):
1812
+ for call in compat_itervalues(function.calls):
1813
+ if call.ratio is not None:
1814
+ callee = profile.functions[call.callee_id]
1815
+ call[TOTAL_TIME_RATIO] = call.ratio * callee[TOTAL_TIME_RATIO];
1816
+ else:
1817
+ assert False
1818
+
1819
+ return profile
1820
+
1821
+ def parse_event(self):
1822
+ if self.eof():
1823
+ return
1824
+
1825
+ line = self.consume()
1826
+ assert line
1827
+
1828
+ callchain = self.parse_callchain()
1829
+ if not callchain:
1830
+ return
1831
+
1832
+ callee = callchain[0]
1833
+ callee[SAMPLES] += 1
1834
+ self.profile[SAMPLES] += 1
1835
+
1836
+ for caller in callchain[1:]:
1837
+ try:
1838
+ call = caller.calls[callee.id]
1839
+ except KeyError:
1840
+ call = Call(callee.id)
1841
+ call[SAMPLES2] = 1
1842
+ caller.add_call(call)
1843
+ else:
1844
+ call[SAMPLES2] += 1
1845
+
1846
+ callee = caller
1847
+
1848
+ # Increment TOTAL_SAMPLES only once on each function.
1849
+ stack = set(callchain)
1850
+ for function in stack:
1851
+ function[TOTAL_SAMPLES] += 1
1852
+
1853
+ def parse_callchain(self):
1854
+ callchain = []
1855
+ while self.lookahead():
1856
+ function = self.parse_call()
1857
+ if function is None:
1858
+ break
1859
+ callchain.append(function)
1860
+ if self.lookahead() == '':
1861
+ self.consume()
1862
+ return callchain
1863
+
1864
+ call_re = re.compile(r'^\s+(?P<address>[0-9a-fA-F]+)\s+(?P<symbol>.*)\s+\((?P<module>[^)]*)\)$')
1865
+
1866
+ def parse_call(self):
1867
+ line = self.consume()
1868
+ mo = self.call_re.match(line)
1869
+ assert mo
1870
+ if not mo:
1871
+ return None
1872
+
1873
+ function_name = mo.group('symbol')
1874
+ if not function_name:
1875
+ function_name = mo.group('address')
1876
+
1877
+ module = mo.group('module')
1878
+
1879
+ function_id = function_name + ':' + module
1880
+
1881
+ try:
1882
+ function = self.profile.functions[function_id]
1883
+ except KeyError:
1884
+ function = Function(function_id, function_name)
1885
+ function.module = os.path.basename(module)
1886
+ function[SAMPLES] = 0
1887
+ function[TOTAL_SAMPLES] = 0
1888
+ self.profile.add_function(function)
1889
+
1890
+ return function
1891
+
1892
+
1893
+ class OprofileParser(LineParser):
1894
+ """Parser for oprofile callgraph output.
1895
+
1896
+ See also:
1897
+ - http://oprofile.sourceforge.net/doc/opreport.html#opreport-callgraph
1898
+ """
1899
+
1900
+ _fields_re = {
1901
+ 'samples': r'(\d+)',
1902
+ '%': r'(\S+)',
1903
+ 'linenr info': r'(?P<source>\(no location information\)|\S+:\d+)',
1904
+ 'image name': r'(?P<image>\S+(?:\s\(tgid:[^)]*\))?)',
1905
+ 'app name': r'(?P<application>\S+)',
1906
+ 'symbol name': r'(?P<symbol>\(no symbols\)|.+?)',
1907
+ }
1908
+
1909
+ def __init__(self, infile):
1910
+ LineParser.__init__(self, infile)
1911
+ self.entries = {}
1912
+ self.entry_re = None
1913
+
1914
+ def add_entry(self, callers, function, callees):
1915
+ try:
1916
+ entry = self.entries[function.id]
1917
+ except KeyError:
1918
+ self.entries[function.id] = (callers, function, callees)
1919
+ else:
1920
+ callers_total, function_total, callees_total = entry
1921
+ self.update_subentries_dict(callers_total, callers)
1922
+ function_total.samples += function.samples
1923
+ self.update_subentries_dict(callees_total, callees)
1924
+
1925
+ def update_subentries_dict(self, totals, partials):
1926
+ for partial in compat_itervalues(partials):
1927
+ try:
1928
+ total = totals[partial.id]
1929
+ except KeyError:
1930
+ totals[partial.id] = partial
1931
+ else:
1932
+ total.samples += partial.samples
1933
+
1934
+ def parse(self):
1935
+ # read lookahead
1936
+ self.readline()
1937
+
1938
+ self.parse_header()
1939
+ while self.lookahead():
1940
+ self.parse_entry()
1941
+
1942
+ profile = Profile()
1943
+
1944
+ reverse_call_samples = {}
1945
+
1946
+ # populate the profile
1947
+ profile[SAMPLES] = 0
1948
+ for _callers, _function, _callees in compat_itervalues(self.entries):
1949
+ function = Function(_function.id, _function.name)
1950
+ function[SAMPLES] = _function.samples
1951
+ profile.add_function(function)
1952
+ profile[SAMPLES] += _function.samples
1953
+
1954
+ if _function.application:
1955
+ function.process = os.path.basename(_function.application)
1956
+ if _function.image:
1957
+ function.module = os.path.basename(_function.image)
1958
+
1959
+ total_callee_samples = 0
1960
+ for _callee in compat_itervalues(_callees):
1961
+ total_callee_samples += _callee.samples
1962
+
1963
+ for _callee in compat_itervalues(_callees):
1964
+ if not _callee.self:
1965
+ call = Call(_callee.id)
1966
+ call[SAMPLES2] = _callee.samples
1967
+ function.add_call(call)
1968
+
1969
+ # compute derived data
1970
+ profile.validate()
1971
+ profile.find_cycles()
1972
+ profile.ratio(TIME_RATIO, SAMPLES)
1973
+ profile.call_ratios(SAMPLES2)
1974
+ profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
1975
+
1976
+ return profile
1977
+
1978
+ def parse_header(self):
1979
+ while not self.match_header():
1980
+ self.consume()
1981
+ line = self.lookahead()
1982
+ fields = re.split(r'\s\s+', line)
1983
+ entry_re = r'^\s*' + r'\s+'.join([self._fields_re[field] for field in fields]) + r'(?P<self>\s+\[self\])?$'
1984
+ self.entry_re = re.compile(entry_re)
1985
+ self.skip_separator()
1986
+
1987
+ def parse_entry(self):
1988
+ callers = self.parse_subentries()
1989
+ if self.match_primary():
1990
+ function = self.parse_subentry()
1991
+ if function is not None:
1992
+ callees = self.parse_subentries()
1993
+ self.add_entry(callers, function, callees)
1994
+ self.skip_separator()
1995
+
1996
+ def parse_subentries(self):
1997
+ subentries = {}
1998
+ while self.match_secondary():
1999
+ subentry = self.parse_subentry()
2000
+ subentries[subentry.id] = subentry
2001
+ return subentries
2002
+
2003
+ def parse_subentry(self):
2004
+ entry = Struct()
2005
+ line = self.consume()
2006
+ mo = self.entry_re.match(line)
2007
+ if not mo:
2008
+ raise ParseError('failed to parse', line)
2009
+ fields = mo.groupdict()
2010
+ entry.samples = int(mo.group(1))
2011
+ if 'source' in fields and fields['source'] != '(no location information)':
2012
+ source = fields['source']
2013
+ filename, lineno = source.split(':')
2014
+ entry.filename = filename
2015
+ entry.lineno = int(lineno)
2016
+ else:
2017
+ source = ''
2018
+ entry.filename = None
2019
+ entry.lineno = None
2020
+ entry.image = fields.get('image', '')
2021
+ entry.application = fields.get('application', '')
2022
+ if 'symbol' in fields and fields['symbol'] != '(no symbols)':
2023
+ entry.symbol = fields['symbol']
2024
+ else:
2025
+ entry.symbol = ''
2026
+ if entry.symbol.startswith('"') and entry.symbol.endswith('"'):
2027
+ entry.symbol = entry.symbol[1:-1]
2028
+ entry.id = ':'.join((entry.application, entry.image, source, entry.symbol))
2029
+ entry.self = fields.get('self', None) != None
2030
+ if entry.self:
2031
+ entry.id += ':self'
2032
+ if entry.symbol:
2033
+ entry.name = entry.symbol
2034
+ else:
2035
+ entry.name = entry.image
2036
+ return entry
2037
+
2038
+ def skip_separator(self):
2039
+ while not self.match_separator():
2040
+ self.consume()
2041
+ self.consume()
2042
+
2043
+ def match_header(self):
2044
+ line = self.lookahead()
2045
+ return line.startswith('samples')
2046
+
2047
+ def match_separator(self):
2048
+ line = self.lookahead()
2049
+ return line == '-'*len(line)
2050
+
2051
+ def match_primary(self):
2052
+ line = self.lookahead()
2053
+ return not line[:1].isspace()
2054
+
2055
+ def match_secondary(self):
2056
+ line = self.lookahead()
2057
+ return line[:1].isspace()
2058
+
2059
+
2060
+ class HProfParser(LineParser):
2061
+ """Parser for java hprof output
2062
+
2063
+ See also:
2064
+ - http://java.sun.com/developer/technicalArticles/Programming/HPROF.html
2065
+ """
2066
+
2067
+ trace_re = re.compile(r'\t(.*)\((.*):(.*)\)')
2068
+ trace_id_re = re.compile(r'^TRACE (\d+):$')
2069
+
2070
+ def __init__(self, infile):
2071
+ LineParser.__init__(self, infile)
2072
+ self.traces = {}
2073
+ self.samples = {}
2074
+
2075
+ def parse(self):
2076
+ # read lookahead
2077
+ self.readline()
2078
+
2079
+ while not self.lookahead().startswith('------'): self.consume()
2080
+ while not self.lookahead().startswith('TRACE '): self.consume()
2081
+
2082
+ self.parse_traces()
2083
+
2084
+ while not self.lookahead().startswith('CPU'):
2085
+ self.consume()
2086
+
2087
+ self.parse_samples()
2088
+
2089
+ # populate the profile
2090
+ profile = Profile()
2091
+ profile[SAMPLES] = 0
2092
+
2093
+ functions = {}
2094
+
2095
+ # build up callgraph
2096
+ for id, trace in compat_iteritems(self.traces):
2097
+ if not id in self.samples: continue
2098
+ mtime = self.samples[id][0]
2099
+ last = None
2100
+
2101
+ for func, file, line in trace:
2102
+ if not func in functions:
2103
+ function = Function(func, func)
2104
+ function[SAMPLES] = 0
2105
+ profile.add_function(function)
2106
+ functions[func] = function
2107
+
2108
+ function = functions[func]
2109
+ # allocate time to the deepest method in the trace
2110
+ if not last:
2111
+ function[SAMPLES] += mtime
2112
+ profile[SAMPLES] += mtime
2113
+ else:
2114
+ c = function.get_call(last)
2115
+ c[SAMPLES2] += mtime
2116
+
2117
+ last = func
2118
+
2119
+ # compute derived data
2120
+ profile.validate()
2121
+ profile.find_cycles()
2122
+ profile.ratio(TIME_RATIO, SAMPLES)
2123
+ profile.call_ratios(SAMPLES2)
2124
+ profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
2125
+
2126
+ return profile
2127
+
2128
+ def parse_traces(self):
2129
+ while self.lookahead().startswith('TRACE '):
2130
+ self.parse_trace()
2131
+
2132
+ def parse_trace(self):
2133
+ l = self.consume()
2134
+ mo = self.trace_id_re.match(l)
2135
+ tid = mo.group(1)
2136
+ last = None
2137
+ trace = []
2138
+
2139
+ while self.lookahead().startswith('\t'):
2140
+ l = self.consume()
2141
+ match = self.trace_re.search(l)
2142
+ if not match:
2143
+ #sys.stderr.write('Invalid line: %s\n' % l)
2144
+ break
2145
+ else:
2146
+ function_name, file, line = match.groups()
2147
+ trace += [(function_name, file, line)]
2148
+
2149
+ self.traces[int(tid)] = trace
2150
+
2151
+ def parse_samples(self):
2152
+ self.consume()
2153
+ self.consume()
2154
+
2155
+ while not self.lookahead().startswith('CPU'):
2156
+ rank, percent_self, percent_accum, count, traceid, method = self.lookahead().split()
2157
+ self.samples[int(traceid)] = (int(count), method)
2158
+ self.consume()
2159
+
2160
+
2161
+ class SysprofParser(XmlParser):
2162
+
2163
+ def __init__(self, stream):
2164
+ XmlParser.__init__(self, stream)
2165
+
2166
+ def parse(self):
2167
+ objects = {}
2168
+ nodes = {}
2169
+
2170
+ self.element_start('profile')
2171
+ while self.token.type == XML_ELEMENT_START:
2172
+ if self.token.name_or_data == 'objects':
2173
+ assert not objects
2174
+ objects = self.parse_items('objects')
2175
+ elif self.token.name_or_data == 'nodes':
2176
+ assert not nodes
2177
+ nodes = self.parse_items('nodes')
2178
+ else:
2179
+ self.parse_value(self.token.name_or_data)
2180
+ self.element_end('profile')
2181
+
2182
+ return self.build_profile(objects, nodes)
2183
+
2184
+ def parse_items(self, name):
2185
+ assert name[-1] == 's'
2186
+ items = {}
2187
+ self.element_start(name)
2188
+ while self.token.type == XML_ELEMENT_START:
2189
+ id, values = self.parse_item(name[:-1])
2190
+ assert id not in items
2191
+ items[id] = values
2192
+ self.element_end(name)
2193
+ return items
2194
+
2195
+ def parse_item(self, name):
2196
+ attrs = self.element_start(name)
2197
+ id = int(attrs['id'])
2198
+ values = self.parse_values()
2199
+ self.element_end(name)
2200
+ return id, values
2201
+
2202
+ def parse_values(self):
2203
+ values = {}
2204
+ while self.token.type == XML_ELEMENT_START:
2205
+ name = self.token.name_or_data
2206
+ value = self.parse_value(name)
2207
+ assert name not in values
2208
+ values[name] = value
2209
+ return values
2210
+
2211
+ def parse_value(self, tag):
2212
+ self.element_start(tag)
2213
+ value = self.character_data()
2214
+ self.element_end(tag)
2215
+ if value.isdigit():
2216
+ return int(value)
2217
+ if value.startswith('"') and value.endswith('"'):
2218
+ return value[1:-1]
2219
+ return value
2220
+
2221
+ def build_profile(self, objects, nodes):
2222
+ profile = Profile()
2223
+
2224
+ profile[SAMPLES] = 0
2225
+ for id, object in compat_iteritems(objects):
2226
+ # Ignore fake objects (process names, modules, "Everything", "kernel", etc.)
2227
+ if object['self'] == 0:
2228
+ continue
2229
+
2230
+ function = Function(id, object['name'])
2231
+ function[SAMPLES] = object['self']
2232
+ profile.add_function(function)
2233
+ profile[SAMPLES] += function[SAMPLES]
2234
+
2235
+ for id, node in compat_iteritems(nodes):
2236
+ # Ignore fake calls
2237
+ if node['self'] == 0:
2238
+ continue
2239
+
2240
+ # Find a non-ignored parent
2241
+ parent_id = node['parent']
2242
+ while parent_id != 0:
2243
+ parent = nodes[parent_id]
2244
+ caller_id = parent['object']
2245
+ if objects[caller_id]['self'] != 0:
2246
+ break
2247
+ parent_id = parent['parent']
2248
+ if parent_id == 0:
2249
+ continue
2250
+
2251
+ callee_id = node['object']
2252
+
2253
+ assert objects[caller_id]['self']
2254
+ assert objects[callee_id]['self']
2255
+
2256
+ function = profile.functions[caller_id]
2257
+
2258
+ samples = node['self']
2259
+ try:
2260
+ call = function.calls[callee_id]
2261
+ except KeyError:
2262
+ call = Call(callee_id)
2263
+ call[SAMPLES2] = samples
2264
+ function.add_call(call)
2265
+ else:
2266
+ call[SAMPLES2] += samples
2267
+
2268
+ # Compute derived events
2269
+ profile.validate()
2270
+ profile.find_cycles()
2271
+ profile.ratio(TIME_RATIO, SAMPLES)
2272
+ profile.call_ratios(SAMPLES2)
2273
+ profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
2274
+
2275
+ return profile
2276
+
2277
+
2278
+ class XPerfParser(Parser):
2279
+ """Parser for CSVs generted by XPerf, from Microsoft Windows Performance Tools.
2280
+ """
2281
+
2282
+ def __init__(self, stream):
2283
+ Parser.__init__(self)
2284
+ self.stream = stream
2285
+ self.profile = Profile()
2286
+ self.profile[SAMPLES] = 0
2287
+ self.column = {}
2288
+
2289
+ def parse(self):
2290
+ import csv
2291
+ reader = csv.reader(
2292
+ self.stream,
2293
+ delimiter = ',',
2294
+ quotechar = None,
2295
+ escapechar = None,
2296
+ doublequote = False,
2297
+ skipinitialspace = True,
2298
+ lineterminator = '\r\n',
2299
+ quoting = csv.QUOTE_NONE)
2300
+ header = True
2301
+ for row in reader:
2302
+ if header:
2303
+ self.parse_header(row)
2304
+ header = False
2305
+ else:
2306
+ self.parse_row(row)
2307
+
2308
+ # compute derived data
2309
+ self.profile.validate()
2310
+ self.profile.find_cycles()
2311
+ self.profile.ratio(TIME_RATIO, SAMPLES)
2312
+ self.profile.call_ratios(SAMPLES2)
2313
+ self.profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
2314
+
2315
+ return self.profile
2316
+
2317
+ def parse_header(self, row):
2318
+ for column in range(len(row)):
2319
+ name = row[column]
2320
+ assert name not in self.column
2321
+ self.column[name] = column
2322
+
2323
+ def parse_row(self, row):
2324
+ fields = {}
2325
+ for name, column in compat_iteritems(self.column):
2326
+ value = row[column]
2327
+ for factory in int, float:
2328
+ try:
2329
+ value = factory(value)
2330
+ except ValueError:
2331
+ pass
2332
+ else:
2333
+ break
2334
+ fields[name] = value
2335
+
2336
+ process = fields['Process Name']
2337
+ symbol = fields['Module'] + '!' + fields['Function']
2338
+ weight = fields['Weight']
2339
+ count = fields['Count']
2340
+
2341
+ if process == 'Idle':
2342
+ return
2343
+
2344
+ function = self.get_function(process, symbol)
2345
+ function[SAMPLES] += weight * count
2346
+ self.profile[SAMPLES] += weight * count
2347
+
2348
+ stack = fields['Stack']
2349
+ if stack != '?':
2350
+ stack = stack.split('/')
2351
+ assert stack[0] == '[Root]'
2352
+ if stack[-1] != symbol:
2353
+ # XXX: some cases the sampled function does not appear in the stack
2354
+ stack.append(symbol)
2355
+ caller = None
2356
+ for symbol in stack[1:]:
2357
+ callee = self.get_function(process, symbol)
2358
+ if caller is not None:
2359
+ try:
2360
+ call = caller.calls[callee.id]
2361
+ except KeyError:
2362
+ call = Call(callee.id)
2363
+ call[SAMPLES2] = count
2364
+ caller.add_call(call)
2365
+ else:
2366
+ call[SAMPLES2] += count
2367
+ caller = callee
2368
+
2369
+ def get_function(self, process, symbol):
2370
+ function_id = process + '!' + symbol
2371
+
2372
+ try:
2373
+ function = self.profile.functions[function_id]
2374
+ except KeyError:
2375
+ module, name = symbol.split('!', 1)
2376
+ function = Function(function_id, name)
2377
+ function.process = process
2378
+ function.module = module
2379
+ function[SAMPLES] = 0
2380
+ self.profile.add_function(function)
2381
+
2382
+ return function
2383
+
2384
+
2385
+ class SleepyParser(Parser):
2386
+ """Parser for GNU gprof output.
2387
+
2388
+ See also:
2389
+ - http://www.codersnotes.com/sleepy/
2390
+ - http://sleepygraph.sourceforge.net/
2391
+ """
2392
+
2393
+ stdinInput = False
2394
+
2395
+ def __init__(self, filename):
2396
+ Parser.__init__(self)
2397
+
2398
+ from zipfile import ZipFile
2399
+
2400
+ self.database = ZipFile(filename)
2401
+
2402
+ self.symbols = {}
2403
+ self.calls = {}
2404
+
2405
+ self.profile = Profile()
2406
+
2407
+ _symbol_re = re.compile(
2408
+ r'^(?P<id>\w+)' +
2409
+ r'\s+"(?P<module>[^"]*)"' +
2410
+ r'\s+"(?P<procname>[^"]*)"' +
2411
+ r'\s+"(?P<sourcefile>[^"]*)"' +
2412
+ r'\s+(?P<sourceline>\d+)$'
2413
+ )
2414
+
2415
+ def openEntry(self, name):
2416
+ # Some versions of verysleepy use lowercase filenames
2417
+ for database_name in self.database.namelist():
2418
+ if name.lower() == database_name.lower():
2419
+ name = database_name
2420
+ break
2421
+
2422
+ return self.database.open(name, 'rU')
2423
+
2424
+ def parse_symbols(self):
2425
+ for line in self.openEntry('Symbols.txt'):
2426
+ line = line.decode('UTF-8')
2427
+
2428
+ mo = self._symbol_re.match(line)
2429
+ if mo:
2430
+ symbol_id, module, procname, sourcefile, sourceline = mo.groups()
2431
+
2432
+ function_id = ':'.join([module, procname])
2433
+
2434
+ try:
2435
+ function = self.profile.functions[function_id]
2436
+ except KeyError:
2437
+ function = Function(function_id, procname)
2438
+ function.module = module
2439
+ function[SAMPLES] = 0
2440
+ self.profile.add_function(function)
2441
+
2442
+ self.symbols[symbol_id] = function
2443
+
2444
+ def parse_callstacks(self):
2445
+ for line in self.openEntry('Callstacks.txt'):
2446
+ line = line.decode('UTF-8')
2447
+
2448
+ fields = line.split()
2449
+ samples = float(fields[0])
2450
+ callstack = fields[1:]
2451
+
2452
+ callstack = [self.symbols[symbol_id] for symbol_id in callstack]
2453
+
2454
+ callee = callstack[0]
2455
+
2456
+ callee[SAMPLES] += samples
2457
+ self.profile[SAMPLES] += samples
2458
+
2459
+ for caller in callstack[1:]:
2460
+ try:
2461
+ call = caller.calls[callee.id]
2462
+ except KeyError:
2463
+ call = Call(callee.id)
2464
+ call[SAMPLES2] = samples
2465
+ caller.add_call(call)
2466
+ else:
2467
+ call[SAMPLES2] += samples
2468
+
2469
+ callee = caller
2470
+
2471
+ def parse(self):
2472
+ profile = self.profile
2473
+ profile[SAMPLES] = 0
2474
+
2475
+ self.parse_symbols()
2476
+ self.parse_callstacks()
2477
+
2478
+ # Compute derived events
2479
+ profile.validate()
2480
+ profile.find_cycles()
2481
+ profile.ratio(TIME_RATIO, SAMPLES)
2482
+ profile.call_ratios(SAMPLES2)
2483
+ profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
2484
+
2485
+ return profile
2486
+
2487
+
2488
+ class AQtimeTable:
2489
+
2490
+ def __init__(self, name, fields):
2491
+ self.name = name
2492
+
2493
+ self.fields = fields
2494
+ self.field_column = {}
2495
+ for column in range(len(fields)):
2496
+ self.field_column[fields[column]] = column
2497
+ self.rows = []
2498
+
2499
+ def __len__(self):
2500
+ return len(self.rows)
2501
+
2502
+ def __iter__(self):
2503
+ for values, children in self.rows:
2504
+ fields = {}
2505
+ for name, value in zip(self.fields, values):
2506
+ fields[name] = value
2507
+ children = dict([(child.name, child) for child in children])
2508
+ yield fields, children
2509
+ raise StopIteration
2510
+
2511
+ def add_row(self, values, children=()):
2512
+ self.rows.append((values, children))
2513
+
2514
+
2515
+ class AQtimeParser(XmlParser):
2516
+
2517
+ def __init__(self, stream):
2518
+ XmlParser.__init__(self, stream)
2519
+ self.tables = {}
2520
+
2521
+ def parse(self):
2522
+ self.element_start('AQtime_Results')
2523
+ self.parse_headers()
2524
+ results = self.parse_results()
2525
+ self.element_end('AQtime_Results')
2526
+ return self.build_profile(results)
2527
+
2528
+ def parse_headers(self):
2529
+ self.element_start('HEADERS')
2530
+ while self.token.type == XML_ELEMENT_START:
2531
+ self.parse_table_header()
2532
+ self.element_end('HEADERS')
2533
+
2534
+ def parse_table_header(self):
2535
+ attrs = self.element_start('TABLE_HEADER')
2536
+ name = attrs['NAME']
2537
+ id = int(attrs['ID'])
2538
+ field_types = []
2539
+ field_names = []
2540
+ while self.token.type == XML_ELEMENT_START:
2541
+ field_type, field_name = self.parse_table_field()
2542
+ field_types.append(field_type)
2543
+ field_names.append(field_name)
2544
+ self.element_end('TABLE_HEADER')
2545
+ self.tables[id] = name, field_types, field_names
2546
+
2547
+ def parse_table_field(self):
2548
+ attrs = self.element_start('TABLE_FIELD')
2549
+ type = attrs['TYPE']
2550
+ name = self.character_data()
2551
+ self.element_end('TABLE_FIELD')
2552
+ return type, name
2553
+
2554
+ def parse_results(self):
2555
+ self.element_start('RESULTS')
2556
+ table = self.parse_data()
2557
+ self.element_end('RESULTS')
2558
+ return table
2559
+
2560
+ def parse_data(self):
2561
+ rows = []
2562
+ attrs = self.element_start('DATA')
2563
+ table_id = int(attrs['TABLE_ID'])
2564
+ table_name, field_types, field_names = self.tables[table_id]
2565
+ table = AQtimeTable(table_name, field_names)
2566
+ while self.token.type == XML_ELEMENT_START:
2567
+ row, children = self.parse_row(field_types)
2568
+ table.add_row(row, children)
2569
+ self.element_end('DATA')
2570
+ return table
2571
+
2572
+ def parse_row(self, field_types):
2573
+ row = [None]*len(field_types)
2574
+ children = []
2575
+ self.element_start('ROW')
2576
+ while self.token.type == XML_ELEMENT_START:
2577
+ if self.token.name_or_data == 'FIELD':
2578
+ field_id, field_value = self.parse_field(field_types)
2579
+ row[field_id] = field_value
2580
+ elif self.token.name_or_data == 'CHILDREN':
2581
+ children = self.parse_children()
2582
+ else:
2583
+ raise XmlTokenMismatch("<FIELD ...> or <CHILDREN ...>", self.token)
2584
+ self.element_end('ROW')
2585
+ return row, children
2586
+
2587
+ def parse_field(self, field_types):
2588
+ attrs = self.element_start('FIELD')
2589
+ id = int(attrs['ID'])
2590
+ type = field_types[id]
2591
+ value = self.character_data()
2592
+ if type == 'Integer':
2593
+ value = int(value)
2594
+ elif type == 'Float':
2595
+ value = float(value)
2596
+ elif type == 'Address':
2597
+ value = int(value)
2598
+ elif type == 'String':
2599
+ pass
2600
+ else:
2601
+ assert False
2602
+ self.element_end('FIELD')
2603
+ return id, value
2604
+
2605
+ def parse_children(self):
2606
+ children = []
2607
+ self.element_start('CHILDREN')
2608
+ while self.token.type == XML_ELEMENT_START:
2609
+ table = self.parse_data()
2610
+ assert table.name not in children
2611
+ children.append(table)
2612
+ self.element_end('CHILDREN')
2613
+ return children
2614
+
2615
+ def build_profile(self, results):
2616
+ assert results.name == 'Routines'
2617
+ profile = Profile()
2618
+ profile[TIME] = 0.0
2619
+ for fields, tables in results:
2620
+ function = self.build_function(fields)
2621
+ children = tables['Children']
2622
+ for fields, _ in children:
2623
+ call = self.build_call(fields)
2624
+ function.add_call(call)
2625
+ profile.add_function(function)
2626
+ profile[TIME] = profile[TIME] + function[TIME]
2627
+ profile[TOTAL_TIME] = profile[TIME]
2628
+ profile.ratio(TOTAL_TIME_RATIO, TOTAL_TIME)
2629
+ return profile
2630
+
2631
+ def build_function(self, fields):
2632
+ function = Function(self.build_id(fields), self.build_name(fields))
2633
+ function[TIME] = fields['Time']
2634
+ function[TOTAL_TIME] = fields['Time with Children']
2635
+ #function[TIME_RATIO] = fields['% Time']/100.0
2636
+ #function[TOTAL_TIME_RATIO] = fields['% with Children']/100.0
2637
+ return function
2638
+
2639
+ def build_call(self, fields):
2640
+ call = Call(self.build_id(fields))
2641
+ call[TIME] = fields['Time']
2642
+ call[TOTAL_TIME] = fields['Time with Children']
2643
+ #call[TIME_RATIO] = fields['% Time']/100.0
2644
+ #call[TOTAL_TIME_RATIO] = fields['% with Children']/100.0
2645
+ return call
2646
+
2647
+ def build_id(self, fields):
2648
+ return ':'.join([fields['Module Name'], fields['Unit Name'], fields['Routine Name']])
2649
+
2650
+ def build_name(self, fields):
2651
+ # TODO: use more fields
2652
+ return fields['Routine Name']
2653
+
2654
+
2655
+ class PstatsParser:
2656
+ """Parser python profiling statistics saved with te pstats module."""
2657
+
2658
+ stdinInput = False
2659
+ multipleInput = True
2660
+
2661
+ def __init__(self, *filename):
2662
+ import pstats
2663
+ try:
2664
+ self.stats = pstats.Stats(*filename)
2665
+ except ValueError:
2666
+ if sys.version_info[0] >= 3:
2667
+ raise
2668
+ import hotshot.stats
2669
+ self.stats = hotshot.stats.load(filename[0])
2670
+ self.profile = Profile()
2671
+ self.function_ids = {}
2672
+
2673
+ def get_function_name(self, key):
2674
+ filename, line, name = key
2675
+ module = os.path.splitext(filename)[0]
2676
+ module = os.path.basename(module)
2677
+ return "%s:%d:%s" % (module, line, name)
2678
+
2679
+ def get_function(self, key):
2680
+ try:
2681
+ id = self.function_ids[key]
2682
+ except KeyError:
2683
+ id = len(self.function_ids)
2684
+ name = self.get_function_name(key)
2685
+ function = Function(id, name)
2686
+ self.profile.functions[id] = function
2687
+ self.function_ids[key] = id
2688
+ else:
2689
+ function = self.profile.functions[id]
2690
+ return function
2691
+
2692
+ def parse(self):
2693
+ self.profile[TIME] = 0.0
2694
+ self.profile[TOTAL_TIME] = self.stats.total_tt
2695
+ for fn, (cc, nc, tt, ct, callers) in compat_iteritems(self.stats.stats):
2696
+ callee = self.get_function(fn)
2697
+ callee.called = nc
2698
+ callee[TOTAL_TIME] = ct
2699
+ callee[TIME] = tt
2700
+ self.profile[TIME] += tt
2701
+ self.profile[TOTAL_TIME] = max(self.profile[TOTAL_TIME], ct)
2702
+ for fn, value in compat_iteritems(callers):
2703
+ caller = self.get_function(fn)
2704
+ call = Call(callee.id)
2705
+ if isinstance(value, tuple):
2706
+ for i in xrange(0, len(value), 4):
2707
+ nc, cc, tt, ct = value[i:i+4]
2708
+ if CALLS in call:
2709
+ call[CALLS] += cc
2710
+ else:
2711
+ call[CALLS] = cc
2712
+
2713
+ if TOTAL_TIME in call:
2714
+ call[TOTAL_TIME] += ct
2715
+ else:
2716
+ call[TOTAL_TIME] = ct
2717
+
2718
+ else:
2719
+ call[CALLS] = value
2720
+ call[TOTAL_TIME] = ratio(value, nc)*ct
2721
+
2722
+ caller.add_call(call)
2723
+ #self.stats.print_stats()
2724
+ #self.stats.print_callees()
2725
+
2726
+ # Compute derived events
2727
+ self.profile.validate()
2728
+ self.profile.ratio(TIME_RATIO, TIME)
2729
+ self.profile.ratio(TOTAL_TIME_RATIO, TOTAL_TIME)
2730
+
2731
+ return self.profile
2732
+
2733
+
2734
+ class Theme:
2735
+
2736
+ def __init__(self,
2737
+ bgcolor = (0.0, 0.0, 1.0),
2738
+ mincolor = (0.0, 0.0, 0.0),
2739
+ maxcolor = (0.0, 0.0, 1.0),
2740
+ fontname = "Arial",
2741
+ fontcolor = "white",
2742
+ nodestyle = "filled",
2743
+ minfontsize = 10.0,
2744
+ maxfontsize = 10.0,
2745
+ minpenwidth = 0.5,
2746
+ maxpenwidth = 4.0,
2747
+ gamma = 2.2,
2748
+ skew = 1.0):
2749
+ self.bgcolor = bgcolor
2750
+ self.mincolor = mincolor
2751
+ self.maxcolor = maxcolor
2752
+ self.fontname = fontname
2753
+ self.fontcolor = fontcolor
2754
+ self.nodestyle = nodestyle
2755
+ self.minfontsize = minfontsize
2756
+ self.maxfontsize = maxfontsize
2757
+ self.minpenwidth = minpenwidth
2758
+ self.maxpenwidth = maxpenwidth
2759
+ self.gamma = gamma
2760
+ self.skew = skew
2761
+
2762
+ def graph_bgcolor(self):
2763
+ return self.hsl_to_rgb(*self.bgcolor)
2764
+
2765
+ def graph_fontname(self):
2766
+ return self.fontname
2767
+
2768
+ def graph_fontcolor(self):
2769
+ return self.fontcolor
2770
+
2771
+ def graph_fontsize(self):
2772
+ return self.minfontsize
2773
+
2774
+ def node_bgcolor(self, weight):
2775
+ return self.color(weight)
2776
+
2777
+ def node_fgcolor(self, weight):
2778
+ if self.nodestyle == "filled":
2779
+ return self.graph_bgcolor()
2780
+ else:
2781
+ return self.color(weight)
2782
+
2783
+ def node_fontsize(self, weight):
2784
+ return self.fontsize(weight)
2785
+
2786
+ def node_style(self):
2787
+ return self.nodestyle
2788
+
2789
+ def edge_color(self, weight):
2790
+ return self.color(weight)
2791
+
2792
+ def edge_fontsize(self, weight):
2793
+ return self.fontsize(weight)
2794
+
2795
+ def edge_penwidth(self, weight):
2796
+ return max(weight*self.maxpenwidth, self.minpenwidth)
2797
+
2798
+ def edge_arrowsize(self, weight):
2799
+ return 0.5 * math.sqrt(self.edge_penwidth(weight))
2800
+
2801
+ def fontsize(self, weight):
2802
+ return max(weight**2 * self.maxfontsize, self.minfontsize)
2803
+
2804
+ def color(self, weight):
2805
+ weight = min(max(weight, 0.0), 1.0)
2806
+
2807
+ hmin, smin, lmin = self.mincolor
2808
+ hmax, smax, lmax = self.maxcolor
2809
+
2810
+ if self.skew < 0:
2811
+ raise ValueError("Skew must be greater than 0")
2812
+ elif self.skew == 1.0:
2813
+ h = hmin + weight*(hmax - hmin)
2814
+ s = smin + weight*(smax - smin)
2815
+ l = lmin + weight*(lmax - lmin)
2816
+ else:
2817
+ base = self.skew
2818
+ h = hmin + ((hmax-hmin)*(-1.0 + (base ** weight)) / (base - 1.0))
2819
+ s = smin + ((smax-smin)*(-1.0 + (base ** weight)) / (base - 1.0))
2820
+ l = lmin + ((lmax-lmin)*(-1.0 + (base ** weight)) / (base - 1.0))
2821
+
2822
+ return self.hsl_to_rgb(h, s, l)
2823
+
2824
+ def hsl_to_rgb(self, h, s, l):
2825
+ """Convert a color from HSL color-model to RGB.
2826
+
2827
+ See also:
2828
+ - http://www.w3.org/TR/css3-color/#hsl-color
2829
+ """
2830
+
2831
+ h = h % 1.0
2832
+ s = min(max(s, 0.0), 1.0)
2833
+ l = min(max(l, 0.0), 1.0)
2834
+
2835
+ if l <= 0.5:
2836
+ m2 = l*(s + 1.0)
2837
+ else:
2838
+ m2 = l + s - l*s
2839
+ m1 = l*2.0 - m2
2840
+ r = self._hue_to_rgb(m1, m2, h + 1.0/3.0)
2841
+ g = self._hue_to_rgb(m1, m2, h)
2842
+ b = self._hue_to_rgb(m1, m2, h - 1.0/3.0)
2843
+
2844
+ # Apply gamma correction
2845
+ r **= self.gamma
2846
+ g **= self.gamma
2847
+ b **= self.gamma
2848
+
2849
+ return (r, g, b)
2850
+
2851
+ def _hue_to_rgb(self, m1, m2, h):
2852
+ if h < 0.0:
2853
+ h += 1.0
2854
+ elif h > 1.0:
2855
+ h -= 1.0
2856
+ if h*6 < 1.0:
2857
+ return m1 + (m2 - m1)*h*6.0
2858
+ elif h*2 < 1.0:
2859
+ return m2
2860
+ elif h*3 < 2.0:
2861
+ return m1 + (m2 - m1)*(2.0/3.0 - h)*6.0
2862
+ else:
2863
+ return m1
2864
+
2865
+
2866
+ TEMPERATURE_COLORMAP = Theme(
2867
+ mincolor = (2.0/3.0, 0.80, 0.25), # dark blue
2868
+ maxcolor = (0.0, 1.0, 0.5), # satured red
2869
+ gamma = 1.0
2870
+ )
2871
+
2872
+ PINK_COLORMAP = Theme(
2873
+ mincolor = (0.0, 1.0, 0.90), # pink
2874
+ maxcolor = (0.0, 1.0, 0.5), # satured red
2875
+ )
2876
+
2877
+ GRAY_COLORMAP = Theme(
2878
+ mincolor = (0.0, 0.0, 0.85), # light gray
2879
+ maxcolor = (0.0, 0.0, 0.0), # black
2880
+ )
2881
+
2882
+ BW_COLORMAP = Theme(
2883
+ minfontsize = 8.0,
2884
+ maxfontsize = 24.0,
2885
+ mincolor = (0.0, 0.0, 0.0), # black
2886
+ maxcolor = (0.0, 0.0, 0.0), # black
2887
+ minpenwidth = 0.1,
2888
+ maxpenwidth = 8.0,
2889
+ )
2890
+
2891
+ PRINT_COLORMAP = Theme(
2892
+ minfontsize = 18.0,
2893
+ maxfontsize = 30.0,
2894
+ fontcolor = "black",
2895
+ nodestyle = "solid",
2896
+ mincolor = (0.0, 0.0, 0.0), # black
2897
+ maxcolor = (0.0, 0.0, 0.0), # black
2898
+ minpenwidth = 0.1,
2899
+ maxpenwidth = 8.0,
2900
+ )
2901
+
2902
+
2903
+ class DotWriter:
2904
+ """Writer for the DOT language.
2905
+
2906
+ See also:
2907
+ - "The DOT Language" specification
2908
+ http://www.graphviz.org/doc/info/lang.html
2909
+ """
2910
+
2911
+ strip = False
2912
+ wrap = False
2913
+
2914
+ def __init__(self, fp):
2915
+ self.fp = fp
2916
+
2917
+ def wrap_function_name(self, name):
2918
+ """Split the function name on multiple lines."""
2919
+
2920
+ if len(name) > 32:
2921
+ ratio = 2.0/3.0
2922
+ height = max(int(len(name)/(1.0 - ratio) + 0.5), 1)
2923
+ width = max(len(name)/height, 32)
2924
+ # TODO: break lines in symbols
2925
+ name = textwrap.fill(name, width, break_long_words=False)
2926
+
2927
+ # Take away spaces
2928
+ name = name.replace(", ", ",")
2929
+ name = name.replace("> >", ">>")
2930
+ name = name.replace("> >", ">>") # catch consecutive
2931
+
2932
+ return name
2933
+
2934
+ show_function_events = [TOTAL_TIME_RATIO, TIME_RATIO]
2935
+ show_edge_events = [TOTAL_TIME_RATIO, CALLS]
2936
+
2937
+ def graph(self, profile, theme):
2938
+ self.begin_graph()
2939
+
2940
+ fontname = theme.graph_fontname()
2941
+ fontcolor = theme.graph_fontcolor()
2942
+ nodestyle = theme.node_style()
2943
+
2944
+ self.attr('graph', fontname=fontname, ranksep=0.25, nodesep=0.125)
2945
+ self.attr('node', fontname=fontname, shape="box", style=nodestyle, fontcolor=fontcolor, width=0, height=0)
2946
+ self.attr('edge', fontname=fontname)
2947
+
2948
+ for function in compat_itervalues(profile.functions):
2949
+ labels = []
2950
+ if function.process is not None:
2951
+ labels.append(function.process)
2952
+ if function.module is not None:
2953
+ labels.append(function.module)
2954
+
2955
+ if self.strip:
2956
+ function_name = function.stripped_name()
2957
+ else:
2958
+ function_name = function.name
2959
+ if self.wrap:
2960
+ function_name = self.wrap_function_name(function_name)
2961
+ labels.append(function_name)
2962
+
2963
+ for event in self.show_function_events:
2964
+ if event in function.events:
2965
+ label = event.format(function[event])
2966
+ labels.append(label)
2967
+ if function.called is not None:
2968
+ labels.append("%u%s" % (function.called, MULTIPLICATION_SIGN))
2969
+
2970
+ if function.weight is not None:
2971
+ weight = function.weight
2972
+ else:
2973
+ weight = 0.0
2974
+
2975
+ label = '\n'.join(labels)
2976
+ self.node(function.id,
2977
+ label = label,
2978
+ color = self.color(theme.node_bgcolor(weight)),
2979
+ fontcolor = self.color(theme.node_fgcolor(weight)),
2980
+ fontsize = "%.2f" % theme.node_fontsize(weight),
2981
+ )
2982
+
2983
+ for call in compat_itervalues(function.calls):
2984
+ callee = profile.functions[call.callee_id]
2985
+
2986
+ labels = []
2987
+ for event in self.show_edge_events:
2988
+ if event in call.events:
2989
+ label = event.format(call[event])
2990
+ labels.append(label)
2991
+
2992
+ if call.weight is not None:
2993
+ weight = call.weight
2994
+ elif callee.weight is not None:
2995
+ weight = callee.weight
2996
+ else:
2997
+ weight = 0.0
2998
+
2999
+ label = '\n'.join(labels)
3000
+
3001
+ self.edge(function.id, call.callee_id,
3002
+ label = label,
3003
+ color = self.color(theme.edge_color(weight)),
3004
+ fontcolor = self.color(theme.edge_color(weight)),
3005
+ fontsize = "%.2f" % theme.edge_fontsize(weight),
3006
+ penwidth = "%.2f" % theme.edge_penwidth(weight),
3007
+ labeldistance = "%.2f" % theme.edge_penwidth(weight),
3008
+ arrowsize = "%.2f" % theme.edge_arrowsize(weight),
3009
+ )
3010
+
3011
+ self.end_graph()
3012
+
3013
+ def begin_graph(self):
3014
+ self.write('digraph {\n')
3015
+
3016
+ def end_graph(self):
3017
+ self.write('}\n')
3018
+
3019
+ def attr(self, what, **attrs):
3020
+ self.write("\t")
3021
+ self.write(what)
3022
+ self.attr_list(attrs)
3023
+ self.write(";\n")
3024
+
3025
+ def node(self, node, **attrs):
3026
+ self.write("\t")
3027
+ self.id(node)
3028
+ self.attr_list(attrs)
3029
+ self.write(";\n")
3030
+
3031
+ def edge(self, src, dst, **attrs):
3032
+ self.write("\t")
3033
+ self.id(src)
3034
+ self.write(" -> ")
3035
+ self.id(dst)
3036
+ self.attr_list(attrs)
3037
+ self.write(";\n")
3038
+
3039
+ def attr_list(self, attrs):
3040
+ if not attrs:
3041
+ return
3042
+ self.write(' [')
3043
+ first = True
3044
+ for name, value in compat_iteritems(attrs):
3045
+ if first:
3046
+ first = False
3047
+ else:
3048
+ self.write(", ")
3049
+ self.id(name)
3050
+ self.write('=')
3051
+ self.id(value)
3052
+ self.write(']')
3053
+
3054
+ def id(self, id):
3055
+ if isinstance(id, (int, float)):
3056
+ s = str(id)
3057
+ elif isinstance(id, basestring):
3058
+ if id.isalnum() and not id.startswith('0x'):
3059
+ s = id
3060
+ else:
3061
+ s = self.escape(id)
3062
+ else:
3063
+ raise TypeError
3064
+ self.write(s)
3065
+
3066
+ def color(self, rgb):
3067
+ r, g, b = rgb
3068
+
3069
+ def float2int(f):
3070
+ if f <= 0.0:
3071
+ return 0
3072
+ if f >= 1.0:
3073
+ return 255
3074
+ return int(255.0*f + 0.5)
3075
+
3076
+ return "#" + "".join(["%02x" % float2int(c) for c in (r, g, b)])
3077
+
3078
+ def escape(self, s):
3079
+ if not PYTHON_3:
3080
+ s = s.encode('utf-8')
3081
+ s = s.replace('\\', r'\\')
3082
+ s = s.replace('\n', r'\n')
3083
+ s = s.replace('\t', r'\t')
3084
+ s = s.replace('"', r'\"')
3085
+ return '"' + s + '"'
3086
+
3087
+ def write(self, s):
3088
+ self.fp.write(s)
3089
+
3090
+
3091
+ class Main:
3092
+ """Main program."""
3093
+
3094
+ themes = {
3095
+ "color": TEMPERATURE_COLORMAP,
3096
+ "pink": PINK_COLORMAP,
3097
+ "gray": GRAY_COLORMAP,
3098
+ "bw": BW_COLORMAP,
3099
+ "print": PRINT_COLORMAP,
3100
+ }
3101
+
3102
+ formats = {
3103
+ "aqtime": AQtimeParser,
3104
+ "axe": AXEParser,
3105
+ "callgrind": CallgrindParser,
3106
+ "hprof": HProfParser,
3107
+ "oprofile": OprofileParser,
3108
+ "perf": PerfParser,
3109
+ "prof": GprofParser,
3110
+ "pstats": PstatsParser,
3111
+ "sleepy": SleepyParser,
3112
+ "sysprof": SysprofParser,
3113
+ "xperf": XPerfParser,
3114
+ }
3115
+
3116
+ def naturalJoin(self, values):
3117
+ if len(values) >= 2:
3118
+ return ', '.join(values[:-1]) + ' or ' + values[-1]
3119
+
3120
+ else:
3121
+ return ''.join(values)
3122
+
3123
+ def main(self):
3124
+ """Main program."""
3125
+
3126
+ global totalMethod
3127
+
3128
+ formatNames = list(self.formats.keys())
3129
+ formatNames.sort()
3130
+
3131
+ optparser = optparse.OptionParser(
3132
+ usage="\n\t%prog [options] [file] ...")
3133
+ optparser.add_option(
3134
+ '-o', '--output', metavar='FILE',
3135
+ type="string", dest="output",
3136
+ help="output filename [stdout]")
3137
+ optparser.add_option(
3138
+ '-n', '--node-thres', metavar='PERCENTAGE',
3139
+ type="float", dest="node_thres", default=0.5,
3140
+ help="eliminate nodes below this threshold [default: %default]")
3141
+ optparser.add_option(
3142
+ '-e', '--edge-thres', metavar='PERCENTAGE',
3143
+ type="float", dest="edge_thres", default=0.1,
3144
+ help="eliminate edges below this threshold [default: %default]")
3145
+ optparser.add_option(
3146
+ '-f', '--format',
3147
+ type="choice", choices=formatNames,
3148
+ dest="format", default="prof",
3149
+ help="profile format: %s [default: %%default]" % self.naturalJoin(formatNames))
3150
+ optparser.add_option(
3151
+ '--total',
3152
+ type="choice", choices=('callratios', 'callstacks'),
3153
+ dest="totalMethod", default=totalMethod,
3154
+ help="preferred method of calculating total time: callratios or callstacks (currently affects only perf format) [default: %default]")
3155
+ optparser.add_option(
3156
+ '-c', '--colormap',
3157
+ type="choice", choices=('color', 'pink', 'gray', 'bw', 'print'),
3158
+ dest="theme", default="color",
3159
+ help="color map: color, pink, gray, bw, or print [default: %default]")
3160
+ optparser.add_option(
3161
+ '-s', '--strip',
3162
+ action="store_true",
3163
+ dest="strip", default=False,
3164
+ help="strip function parameters, template parameters, and const modifiers from demangled C++ function names")
3165
+ optparser.add_option(
3166
+ '-w', '--wrap',
3167
+ action="store_true",
3168
+ dest="wrap", default=False,
3169
+ help="wrap function names")
3170
+ optparser.add_option(
3171
+ '--show-samples',
3172
+ action="store_true",
3173
+ dest="show_samples", default=False,
3174
+ help="show function samples")
3175
+ # add option to create subtree or show paths
3176
+ optparser.add_option(
3177
+ '-z', '--root',
3178
+ type="string",
3179
+ dest="root", default="",
3180
+ help="prune call graph to show only descendants of specified root function")
3181
+ optparser.add_option(
3182
+ '-l', '--leaf',
3183
+ type="string",
3184
+ dest="leaf", default="",
3185
+ help="prune call graph to show only ancestors of specified leaf function")
3186
+ # add a new option to control skew of the colorization curve
3187
+ optparser.add_option(
3188
+ '--skew',
3189
+ type="float", dest="theme_skew", default=1.0,
3190
+ help="skew the colorization curve. Values < 1.0 give more variety to lower percentages. Values > 1.0 give less variety to lower percentages")
3191
+ (self.options, self.args) = optparser.parse_args(sys.argv[1:])
3192
+
3193
+ if len(self.args) > 1 and self.options.format != 'pstats':
3194
+ optparser.error('incorrect number of arguments')
3195
+
3196
+ try:
3197
+ self.theme = self.themes[self.options.theme]
3198
+ except KeyError:
3199
+ optparser.error('invalid colormap \'%s\'' % self.options.theme)
3200
+
3201
+ # set skew on the theme now that it has been picked.
3202
+ if self.options.theme_skew:
3203
+ self.theme.skew = self.options.theme_skew
3204
+
3205
+ totalMethod = self.options.totalMethod
3206
+
3207
+ try:
3208
+ Format = self.formats[self.options.format]
3209
+ except KeyError:
3210
+ optparser.error('invalid format \'%s\'' % self.options.format)
3211
+
3212
+ if Format.stdinInput:
3213
+ if not self.args:
3214
+ fp = sys.stdin
3215
+ else:
3216
+ fp = open(self.args[0], 'rt')
3217
+ parser = Format(fp)
3218
+ elif Format.multipleInput:
3219
+ if not self.args:
3220
+ optparser.error('at least a file must be specified for %s input' % self.options.format)
3221
+ parser = Format(*self.args)
3222
+ else:
3223
+ if len(self.args) != 1:
3224
+ optparser.error('exactly one file must be specified for %s input' % self.options.format)
3225
+ parser = Format(self.args[0])
3226
+
3227
+ self.profile = parser.parse()
3228
+
3229
+ if self.options.output is None:
3230
+ self.output = sys.stdout
3231
+ else:
3232
+ if PYTHON_3:
3233
+ self.output = open(self.options.output, 'wt', encoding='UTF-8')
3234
+ else:
3235
+ self.output = open(self.options.output, 'wt')
3236
+
3237
+ self.write_graph()
3238
+
3239
+ def write_graph(self):
3240
+ dot = DotWriter(self.output)
3241
+ dot.strip = self.options.strip
3242
+ dot.wrap = self.options.wrap
3243
+ if self.options.show_samples:
3244
+ dot.show_function_events.append(SAMPLES)
3245
+
3246
+ profile = self.profile
3247
+ profile.prune(self.options.node_thres/100.0, self.options.edge_thres/100.0)
3248
+
3249
+ if self.options.root:
3250
+ rootId = profile.getFunctionId(self.options.root)
3251
+ if not rootId:
3252
+ sys.stderr.write('root node ' + self.options.root + ' not found (might already be pruned : try -e0 -n0 flags)\n')
3253
+ sys.exit(1)
3254
+ profile.prune_root(rootId)
3255
+ if self.options.leaf:
3256
+ leafId = profile.getFunctionId(self.options.leaf)
3257
+ if not leafId:
3258
+ sys.stderr.write('leaf node ' + self.options.leaf + ' not found (maybe already pruned : try -e0 -n0 flags)\n')
3259
+ sys.exit(1)
3260
+ profile.prune_leaf(leafId)
3261
+
3262
+ dot.graph(profile, self.theme)
3263
+
3264
+
3265
+ if __name__ == '__main__':
3266
+ Main().main()