tango-etl 0.0.1 → 0.1.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (44) hide show
  1. checksums.yaml +4 -4
  2. data/{changelog.md → CHANGELOG.md} +5 -1
  3. data/{readme.md → README.md} +1 -1
  4. data/doc/Multidb/Balancer.html +202 -0
  5. data/doc/Multidb.html +184 -0
  6. data/doc/Tango/AbstractModel.html +608 -0
  7. data/doc/Tango/App.html +1334 -0
  8. data/doc/Tango/DatabaseLocker.html +541 -0
  9. data/doc/Tango/ETL/Dispatcher.html +465 -0
  10. data/doc/Tango/ETL/HandlerInterface.html +639 -0
  11. data/doc/Tango/ETL/OperatorInterface.html +494 -0
  12. data/doc/Tango/ETL.html +117 -0
  13. data/doc/Tango/Kernel.html +432 -0
  14. data/doc/Tango/LinkStack.html +737 -0
  15. data/doc/Tango/Resource/Buffer.html +565 -0
  16. data/doc/Tango/Resource/Cache.html +834 -0
  17. data/doc/Tango/Resource.html +117 -0
  18. data/doc/Tango.html +131 -0
  19. data/doc/_index.html +297 -0
  20. data/doc/class_list.html +54 -0
  21. data/doc/css/common.css +1 -0
  22. data/doc/css/full_list.css +57 -0
  23. data/doc/css/style.css +339 -0
  24. data/doc/file.README.html +80 -0
  25. data/doc/file_list.html +56 -0
  26. data/doc/frames.html +26 -0
  27. data/doc/index.html +80 -0
  28. data/doc/js/app.js +219 -0
  29. data/doc/js/full_list.js +178 -0
  30. data/doc/js/jquery.js +4 -0
  31. data/doc/method_list.html +365 -0
  32. data/doc/top-level-namespace.html +112 -0
  33. data/lib/tango/abstract_model.rb +1 -0
  34. data/lib/tango/app.rb +4 -7
  35. data/lib/tango/database_locker.rb +23 -2
  36. data/lib/tango/etl/handler_interface.rb +12 -6
  37. data/lib/tango/resource/buffer.rb +7 -7
  38. data/lib/tango/resource/cache.rb +18 -13
  39. data/lib/tango/version.rb +1 -1
  40. data/test/support/lib/simple_handler.rb +5 -14
  41. data/test/unit/etl/test_dispatcher.rb +21 -5
  42. metadata +33 -6
  43. data/config/app.yml.sample +0 -6
  44. data/config/database.yml.sample +0 -27
data/doc/Tango/App.html ADDED
@@ -0,0 +1,1334 @@
1
+ <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
2
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
3
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
4
+ <head>
5
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
6
+ <title>
7
+ Class: Tango::App
8
+
9
+ &mdash; Documentation by YARD 0.8.7.4
10
+
11
+ </title>
12
+
13
+ <link rel="stylesheet" href="../css/style.css" type="text/css" charset="utf-8" />
14
+
15
+ <link rel="stylesheet" href="../css/common.css" type="text/css" charset="utf-8" />
16
+
17
+ <script type="text/javascript" charset="utf-8">
18
+ hasFrames = window.top.frames.main ? true : false;
19
+ relpath = '../';
20
+ framesUrl = "../frames.html#!Tango/App.html";
21
+ </script>
22
+
23
+
24
+ <script type="text/javascript" charset="utf-8" src="../js/jquery.js"></script>
25
+
26
+ <script type="text/javascript" charset="utf-8" src="../js/app.js"></script>
27
+
28
+
29
+ </head>
30
+ <body>
31
+ <div id="header">
32
+ <div id="menu">
33
+
34
+ <a href="../_index.html">Index (A)</a> &raquo;
35
+ <span class='title'><span class='object_link'><a href="../Tango.html" title="Tango (module)">Tango</a></span></span>
36
+ &raquo;
37
+ <span class="title">App</span>
38
+
39
+
40
+ <div class="noframes"><span class="title">(</span><a href="." target="_top">no frames</a><span class="title">)</span></div>
41
+ </div>
42
+
43
+ <div id="search">
44
+
45
+ <a class="full_list_link" id="class_list_link"
46
+ href="../class_list.html">
47
+ Class List
48
+ </a>
49
+
50
+ <a class="full_list_link" id="method_list_link"
51
+ href="../method_list.html">
52
+ Method List
53
+ </a>
54
+
55
+ <a class="full_list_link" id="file_list_link"
56
+ href="../file_list.html">
57
+ File List
58
+ </a>
59
+
60
+ </div>
61
+ <div class="clear"></div>
62
+ </div>
63
+
64
+ <iframe id="search_frame"></iframe>
65
+
66
+ <div id="content"><h1>Class: Tango::App
67
+
68
+
69
+
70
+ </h1>
71
+
72
+ <dl class="box">
73
+
74
+ <dt class="r1">Inherits:</dt>
75
+ <dd class="r1">
76
+ <span class="inheritName">Object</span>
77
+
78
+ <ul class="fullTree">
79
+ <li>Object</li>
80
+
81
+ <li class="next">Tango::App</li>
82
+
83
+ </ul>
84
+ <a href="#" class="inheritanceTree">show all</a>
85
+
86
+ </dd>
87
+
88
+
89
+
90
+
91
+
92
+
93
+
94
+
95
+
96
+ <dt class="r2 last">Defined in:</dt>
97
+ <dd class="r2 last">lib/tango/app.rb</dd>
98
+
99
+ </dl>
100
+ <div class="clear"></div>
101
+
102
+ <h2>Overview</h2><div class="docstring">
103
+ <div class="discussion">
104
+
105
+ <p>Tango application</p>
106
+
107
+
108
+ </div>
109
+ </div>
110
+ <div class="tags">
111
+
112
+ <p class="tag_title">Author:</p>
113
+ <ul class="author">
114
+
115
+ <li>
116
+
117
+
118
+
119
+
120
+
121
+ <div class='inline'>
122
+ <p>Mckomo</p>
123
+ </div>
124
+
125
+ </li>
126
+
127
+ </ul>
128
+
129
+ </div>
130
+
131
+
132
+
133
+ <h2>Instance Attribute Summary <small>(<a href="#" class="summary_toggle">collapse</a>)</small></h2>
134
+ <ul class="summary">
135
+
136
+ <li class="public ">
137
+ <span class="summary_signature">
138
+
139
+ <a href="#config-instance_method" title="#config (instance method)">- (Object) <strong>config</strong> </a>
140
+
141
+
142
+
143
+ </span>
144
+
145
+
146
+
147
+
148
+ <span class="note title readonly">readonly</span>
149
+
150
+
151
+
152
+
153
+
154
+
155
+
156
+
157
+
158
+ <span class="summary_desc"><div class='inline'>
159
+ <p>Returns the value of attribute config.</p>
160
+ </div></span>
161
+
162
+ </li>
163
+
164
+
165
+ <li class="public ">
166
+ <span class="summary_signature">
167
+
168
+ <a href="#dispatcher-instance_method" title="#dispatcher (instance method)">- (Object) <strong>dispatcher</strong> </a>
169
+
170
+
171
+
172
+ </span>
173
+
174
+
175
+
176
+
177
+ <span class="note title readonly">readonly</span>
178
+
179
+
180
+
181
+
182
+
183
+
184
+
185
+
186
+
187
+ <span class="summary_desc"><div class='inline'>
188
+ <p>Returns the value of attribute dispatcher.</p>
189
+ </div></span>
190
+
191
+ </li>
192
+
193
+
194
+ <li class="public ">
195
+ <span class="summary_signature">
196
+
197
+ <a href="#link_stack-instance_method" title="#link_stack (instance method)">- (Object) <strong>link_stack</strong> </a>
198
+
199
+
200
+
201
+ </span>
202
+
203
+
204
+
205
+
206
+ <span class="note title readonly">readonly</span>
207
+
208
+
209
+
210
+
211
+
212
+
213
+
214
+
215
+
216
+ <span class="summary_desc"><div class='inline'>
217
+ <p>Returns the value of attribute link_stack.</p>
218
+ </div></span>
219
+
220
+ </li>
221
+
222
+
223
+ <li class="public ">
224
+ <span class="summary_signature">
225
+
226
+ <a href="#logger-instance_method" title="#logger (instance method)">- (Object) <strong>logger</strong> </a>
227
+
228
+
229
+
230
+ </span>
231
+
232
+
233
+
234
+
235
+ <span class="note title readonly">readonly</span>
236
+
237
+
238
+
239
+
240
+
241
+
242
+
243
+
244
+
245
+ <span class="summary_desc"><div class='inline'>
246
+ <p>Returns the value of attribute logger.</p>
247
+ </div></span>
248
+
249
+ </li>
250
+
251
+
252
+ </ul>
253
+
254
+
255
+
256
+
257
+
258
+ <h2>
259
+ Instance Method Summary
260
+ <small>(<a href="#" class="summary_toggle">collapse</a>)</small>
261
+ </h2>
262
+
263
+ <ul class="summary">
264
+
265
+ <li class="public ">
266
+ <span class="summary_signature">
267
+
268
+ <a href="#after-instance_method" title="#after (instance method)">- (Object) <strong>after</strong> </a>
269
+
270
+
271
+
272
+ </span>
273
+
274
+
275
+
276
+
277
+
278
+
279
+
280
+
281
+
282
+ <span class="summary_desc"><div class='inline'>
283
+ <p>Filter run after Tango execution.</p>
284
+ </div></span>
285
+
286
+ </li>
287
+
288
+
289
+ <li class="public ">
290
+ <span class="summary_signature">
291
+
292
+ <a href="#before-instance_method" title="#before (instance method)">- (Object) <strong>before</strong> </a>
293
+
294
+
295
+
296
+ </span>
297
+
298
+
299
+
300
+
301
+
302
+
303
+
304
+
305
+
306
+ <span class="summary_desc"><div class='inline'>
307
+ <p>Filter run before Tango execution.</p>
308
+ </div></span>
309
+
310
+ </li>
311
+
312
+
313
+ <li class="public ">
314
+ <span class="summary_signature">
315
+
316
+ <a href="#initialize-instance_method" title="#initialize (instance method)">- (Tango::App) <strong>initialize</strong>(config: {}, link_stack: nil, dispatcher: nil, cache: nil, http_client: nil, parser: nil, db_locker: nil, logger: nil) </a>
317
+
318
+
319
+
320
+ </span>
321
+
322
+
323
+ <span class="note title constructor">constructor</span>
324
+
325
+
326
+
327
+
328
+
329
+
330
+
331
+
332
+ <span class="summary_desc"><div class='inline'></div></span>
333
+
334
+ </li>
335
+
336
+
337
+ <li class="public ">
338
+ <span class="summary_signature">
339
+
340
+ <a href="#register_model-instance_method" title="#register_model (instance method)">- (Object) <strong>register_model</strong>(symbol, model) </a>
341
+
342
+
343
+
344
+ </span>
345
+
346
+
347
+
348
+
349
+
350
+
351
+
352
+
353
+
354
+ <span class="summary_desc"><div class='inline'>
355
+ <p>Register new resource model.</p>
356
+ </div></span>
357
+
358
+ </li>
359
+
360
+
361
+ <li class="public ">
362
+ <span class="summary_signature">
363
+
364
+ <a href="#register_operator-instance_method" title="#register_operator (instance method)">- (Object) <strong>register_operator</strong>(symbol, operator) </a>
365
+
366
+
367
+
368
+ </span>
369
+
370
+
371
+
372
+
373
+
374
+
375
+
376
+
377
+
378
+ <span class="summary_desc"><div class='inline'>
379
+ <p>Register new resource operator.</p>
380
+ </div></span>
381
+
382
+ </li>
383
+
384
+
385
+ <li class="public ">
386
+ <span class="summary_signature">
387
+
388
+ <a href="#run-instance_method" title="#run (instance method)">- (Integer) <strong>run</strong> </a>
389
+
390
+
391
+
392
+ </span>
393
+
394
+
395
+
396
+
397
+
398
+
399
+
400
+
401
+
402
+ <span class="summary_desc"><div class='inline'>
403
+ <p>Run ETL process.</p>
404
+ </div></span>
405
+
406
+ </li>
407
+
408
+
409
+ </ul>
410
+
411
+
412
+ <div id="constructor_details" class="method_details_list">
413
+ <h2>Constructor Details</h2>
414
+
415
+ <div class="method_details first">
416
+ <h3 class="signature first" id="initialize-instance_method">
417
+
418
+ - (<tt><span class='object_link'><a href="" title="Tango::App (class)">Tango::App</a></span></tt>) <strong>initialize</strong>(config: {}, link_stack: nil, dispatcher: nil, cache: nil, http_client: nil, parser: nil, db_locker: nil, logger: nil)
419
+
420
+
421
+
422
+
423
+
424
+ </h3><div class="docstring">
425
+ <div class="discussion">
426
+
427
+
428
+ </div>
429
+ </div>
430
+ <div class="tags">
431
+ <p class="tag_title">Parameters:</p>
432
+ <ul class="param">
433
+
434
+ <li>
435
+
436
+ <span class='name'>link_stack</span>
437
+
438
+
439
+ <span class='type'>(<tt><span class='object_link'><a href="LinkStack.html" title="Tango::LinkStack (class)">Tango::LinkStack</a></span></tt>)</span>
440
+
441
+
442
+
443
+ </li>
444
+
445
+ <li>
446
+
447
+ <span class='name'>dispatcher</span>
448
+
449
+
450
+ <span class='type'>(<tt>Tango::Etl::Dispatcher</tt>)</span>
451
+
452
+
453
+
454
+ </li>
455
+
456
+ <li>
457
+
458
+ <span class='name'>cache</span>
459
+
460
+
461
+ <span class='type'>(<tt>Tango::Resources::Cache</tt>)</span>
462
+
463
+
464
+
465
+ </li>
466
+
467
+ <li>
468
+
469
+ <span class='name'>http_client</span>
470
+
471
+
472
+ <span class='type'>(<tt>Object</tt>)</span>
473
+
474
+
475
+
476
+ &mdash;
477
+ <div class='inline'>
478
+ <p>Must implement get method</p>
479
+ </div>
480
+
481
+ </li>
482
+
483
+ <li>
484
+
485
+ <span class='name'>parser</span>
486
+
487
+
488
+ <span class='type'>(<tt>Object</tt>)</span>
489
+
490
+
491
+
492
+ &mdash;
493
+ <div class='inline'>
494
+ <p>Must implement parse method</p>
495
+ </div>
496
+
497
+ </li>
498
+
499
+ <li>
500
+
501
+ <span class='name'>db_locker</span>
502
+
503
+
504
+ <span class='type'>(<tt><span class='object_link'><a href="DatabaseLocker.html" title="Tango::DatabaseLocker (class)">DatabaseLocker</a></span></tt>)</span>
505
+
506
+
507
+
508
+ </li>
509
+
510
+ <li>
511
+
512
+ <span class='name'>logger</span>
513
+
514
+
515
+ <span class='type'>(<tt>Logger</tt>)</span>
516
+
517
+
518
+
519
+ </li>
520
+
521
+ </ul>
522
+
523
+
524
+ </div><table class="source_code">
525
+ <tr>
526
+ <td>
527
+ <pre class="lines">
528
+
529
+
530
+ 20
531
+ 21
532
+ 22
533
+ 23
534
+ 24
535
+ 25
536
+ 26
537
+ 27
538
+ 28
539
+ 29
540
+ 30
541
+ 31
542
+ 32
543
+ 33
544
+ 34
545
+ 35
546
+ 36
547
+ 37
548
+ 38</pre>
549
+ </td>
550
+ <td>
551
+ <pre class="code"><span class="info file"># File 'lib/tango/app.rb', line 20</span>
552
+
553
+ <span class='kw'>def</span> <span class='id identifier rubyid_initialize'>initialize</span><span class='lparen'>(</span> <span class='label'>config:</span> <span class='lbrace'>{</span><span class='rbrace'>}</span><span class='comma'>,</span> <span class='label'>link_stack:</span> <span class='kw'>nil</span><span class='comma'>,</span> <span class='label'>dispatcher:</span> <span class='kw'>nil</span><span class='comma'>,</span> <span class='label'>cache:</span> <span class='kw'>nil</span><span class='comma'>,</span> <span class='label'>http_client:</span> <span class='kw'>nil</span><span class='comma'>,</span> <span class='label'>parser:</span> <span class='kw'>nil</span><span class='comma'>,</span> <span class='label'>db_locker:</span> <span class='kw'>nil</span><span class='comma'>,</span> <span class='label'>logger:</span> <span class='kw'>nil</span> <span class='rparen'>)</span>
554
+
555
+ <span class='comment'># Init app properties
556
+ </span> <span class='ivar'>@models</span> <span class='op'>=</span> <span class='lbrace'>{</span><span class='rbrace'>}</span>
557
+ <span class='ivar'>@operators</span> <span class='op'>=</span> <span class='lbrace'>{</span><span class='rbrace'>}</span>
558
+
559
+ <span class='comment'># Set config
560
+ </span> <span class='ivar'>@config</span> <span class='op'>=</span> <span class='id identifier rubyid_config'>config</span>
561
+
562
+ <span class='comment'># Set dependencies
563
+ </span> <span class='ivar'>@link_stack</span> <span class='op'>=</span> <span class='id identifier rubyid_link_stack'>link_stack</span> <span class='op'>||</span> <span class='const'>LinkStack</span><span class='period'>.</span><span class='id identifier rubyid_new'>new</span><span class='lparen'>(</span> <span class='id identifier rubyid_config'>config</span><span class='lbracket'>[</span><span class='tstring'><span class='tstring_beg'>&#39;</span><span class='tstring_content'>target_url</span><span class='tstring_end'>&#39;</span></span><span class='rbracket'>]</span> <span class='rparen'>)</span>
564
+ <span class='ivar'>@dispatcher</span> <span class='op'>=</span> <span class='id identifier rubyid_dispatcher'>dispatcher</span> <span class='op'>||</span> <span class='const'>ETL</span><span class='op'>::</span><span class='const'>Dispatcher</span><span class='period'>.</span><span class='id identifier rubyid_new'>new</span>
565
+ <span class='ivar'>@cache</span> <span class='op'>=</span> <span class='id identifier rubyid_cache'>cache</span> <span class='op'>||</span> <span class='const'>Resource</span><span class='op'>::</span><span class='const'>Cache</span><span class='period'>.</span><span class='id identifier rubyid_new'>new</span><span class='lparen'>(</span> <span class='const'>Resource</span><span class='op'>::</span><span class='const'>Buffer</span><span class='period'>.</span><span class='id identifier rubyid_new'>new</span> <span class='rparen'>)</span>
566
+ <span class='ivar'>@http_client</span> <span class='op'>=</span> <span class='id identifier rubyid_http_client'>http_client</span> <span class='op'>||</span> <span class='const'>HTTParty</span>
567
+ <span class='ivar'>@parser</span> <span class='op'>=</span> <span class='id identifier rubyid_parser'>parser</span> <span class='op'>||</span> <span class='const'>Nokogiri</span><span class='op'>::</span><span class='const'>HTML</span>
568
+ <span class='ivar'>@db_locker</span> <span class='op'>=</span> <span class='id identifier rubyid_db_locker'>db_locker</span> <span class='op'>||</span> <span class='const'>DatabaseLocker</span><span class='period'>.</span><span class='id identifier rubyid_new'>new</span><span class='lparen'>(</span> <span class='const'>Multidb</span><span class='period'>.</span><span class='id identifier rubyid_databases'>databases</span> <span class='rparen'>)</span>
569
+ <span class='ivar'>@logger</span> <span class='op'>=</span> <span class='id identifier rubyid_logger'>logger</span> <span class='op'>||</span> <span class='const'>Logger</span><span class='period'>.</span><span class='id identifier rubyid_new'>new</span><span class='lparen'>(</span> <span class='const'>STDOUT</span> <span class='rparen'>)</span>
570
+
571
+ <span class='kw'>end</span></pre>
572
+ </td>
573
+ </tr>
574
+ </table>
575
+ </div>
576
+
577
+ </div>
578
+
579
+ <div id="instance_attr_details" class="attr_details">
580
+ <h2>Instance Attribute Details</h2>
581
+
582
+
583
+ <span id=""></span>
584
+ <div class="method_details first">
585
+ <h3 class="signature first" id="config-instance_method">
586
+
587
+ - (<tt>Object</tt>) <strong>config</strong> <span class="extras">(readonly)</span>
588
+
589
+
590
+
591
+
592
+
593
+ </h3><div class="docstring">
594
+ <div class="discussion">
595
+
596
+ <p>Returns the value of attribute config</p>
597
+
598
+
599
+ </div>
600
+ </div>
601
+ <div class="tags">
602
+
603
+
604
+ </div><table class="source_code">
605
+ <tr>
606
+ <td>
607
+ <pre class="lines">
608
+
609
+
610
+ 10
611
+ 11
612
+ 12</pre>
613
+ </td>
614
+ <td>
615
+ <pre class="code"><span class="info file"># File 'lib/tango/app.rb', line 10</span>
616
+
617
+ <span class='kw'>def</span> <span class='id identifier rubyid_config'>config</span>
618
+ <span class='ivar'>@config</span>
619
+ <span class='kw'>end</span></pre>
620
+ </td>
621
+ </tr>
622
+ </table>
623
+ </div>
624
+
625
+
626
+ <span id=""></span>
627
+ <div class="method_details ">
628
+ <h3 class="signature " id="dispatcher-instance_method">
629
+
630
+ - (<tt>Object</tt>) <strong>dispatcher</strong> <span class="extras">(readonly)</span>
631
+
632
+
633
+
634
+
635
+
636
+ </h3><div class="docstring">
637
+ <div class="discussion">
638
+
639
+ <p>Returns the value of attribute dispatcher</p>
640
+
641
+
642
+ </div>
643
+ </div>
644
+ <div class="tags">
645
+
646
+
647
+ </div><table class="source_code">
648
+ <tr>
649
+ <td>
650
+ <pre class="lines">
651
+
652
+
653
+ 10
654
+ 11
655
+ 12</pre>
656
+ </td>
657
+ <td>
658
+ <pre class="code"><span class="info file"># File 'lib/tango/app.rb', line 10</span>
659
+
660
+ <span class='kw'>def</span> <span class='id identifier rubyid_dispatcher'>dispatcher</span>
661
+ <span class='ivar'>@dispatcher</span>
662
+ <span class='kw'>end</span></pre>
663
+ </td>
664
+ </tr>
665
+ </table>
666
+ </div>
667
+
668
+
669
+ <span id=""></span>
670
+ <div class="method_details ">
671
+ <h3 class="signature " id="link_stack-instance_method">
672
+
673
+ - (<tt>Object</tt>) <strong>link_stack</strong> <span class="extras">(readonly)</span>
674
+
675
+
676
+
677
+
678
+
679
+ </h3><div class="docstring">
680
+ <div class="discussion">
681
+
682
+ <p>Returns the value of attribute link_stack</p>
683
+
684
+
685
+ </div>
686
+ </div>
687
+ <div class="tags">
688
+
689
+
690
+ </div><table class="source_code">
691
+ <tr>
692
+ <td>
693
+ <pre class="lines">
694
+
695
+
696
+ 10
697
+ 11
698
+ 12</pre>
699
+ </td>
700
+ <td>
701
+ <pre class="code"><span class="info file"># File 'lib/tango/app.rb', line 10</span>
702
+
703
+ <span class='kw'>def</span> <span class='id identifier rubyid_link_stack'>link_stack</span>
704
+ <span class='ivar'>@link_stack</span>
705
+ <span class='kw'>end</span></pre>
706
+ </td>
707
+ </tr>
708
+ </table>
709
+ </div>
710
+
711
+
712
+ <span id=""></span>
713
+ <div class="method_details ">
714
+ <h3 class="signature " id="logger-instance_method">
715
+
716
+ - (<tt>Object</tt>) <strong>logger</strong> <span class="extras">(readonly)</span>
717
+
718
+
719
+
720
+
721
+
722
+ </h3><div class="docstring">
723
+ <div class="discussion">
724
+
725
+ <p>Returns the value of attribute logger</p>
726
+
727
+
728
+ </div>
729
+ </div>
730
+ <div class="tags">
731
+
732
+
733
+ </div><table class="source_code">
734
+ <tr>
735
+ <td>
736
+ <pre class="lines">
737
+
738
+
739
+ 10
740
+ 11
741
+ 12</pre>
742
+ </td>
743
+ <td>
744
+ <pre class="code"><span class="info file"># File 'lib/tango/app.rb', line 10</span>
745
+
746
+ <span class='kw'>def</span> <span class='id identifier rubyid_logger'>logger</span>
747
+ <span class='ivar'>@logger</span>
748
+ <span class='kw'>end</span></pre>
749
+ </td>
750
+ </tr>
751
+ </table>
752
+ </div>
753
+
754
+ </div>
755
+
756
+
757
+ <div id="instance_method_details" class="method_details_list">
758
+ <h2>Instance Method Details</h2>
759
+
760
+
761
+ <div class="method_details first">
762
+ <h3 class="signature first" id="after-instance_method">
763
+
764
+ - (<tt>Object</tt>) <strong>after</strong>
765
+
766
+
767
+
768
+
769
+
770
+ </h3><div class="docstring">
771
+ <div class="discussion">
772
+
773
+ <p>Filter run after Tango execution</p>
774
+
775
+
776
+ </div>
777
+ </div>
778
+ <div class="tags">
779
+
780
+
781
+ </div><table class="source_code">
782
+ <tr>
783
+ <td>
784
+ <pre class="lines">
785
+
786
+
787
+ 45
788
+ 46</pre>
789
+ </td>
790
+ <td>
791
+ <pre class="code"><span class="info file"># File 'lib/tango/app.rb', line 45</span>
792
+
793
+ <span class='kw'>def</span> <span class='id identifier rubyid_after'>after</span>
794
+ <span class='kw'>end</span></pre>
795
+ </td>
796
+ </tr>
797
+ </table>
798
+ </div>
799
+
800
+ <div class="method_details ">
801
+ <h3 class="signature " id="before-instance_method">
802
+
803
+ - (<tt>Object</tt>) <strong>before</strong>
804
+
805
+
806
+
807
+
808
+
809
+ </h3><div class="docstring">
810
+ <div class="discussion">
811
+
812
+ <p>Filter run before Tango execution</p>
813
+
814
+
815
+ </div>
816
+ </div>
817
+ <div class="tags">
818
+
819
+
820
+ </div><table class="source_code">
821
+ <tr>
822
+ <td>
823
+ <pre class="lines">
824
+
825
+
826
+ 41
827
+ 42</pre>
828
+ </td>
829
+ <td>
830
+ <pre class="code"><span class="info file"># File 'lib/tango/app.rb', line 41</span>
831
+
832
+ <span class='kw'>def</span> <span class='id identifier rubyid_before'>before</span>
833
+ <span class='kw'>end</span></pre>
834
+ </td>
835
+ </tr>
836
+ </table>
837
+ </div>
838
+
839
+ <div class="method_details ">
840
+ <h3 class="signature " id="register_model-instance_method">
841
+
842
+ - (<tt>Object</tt>) <strong>register_model</strong>(symbol, model)
843
+
844
+
845
+
846
+
847
+
848
+ </h3><div class="docstring">
849
+ <div class="discussion">
850
+
851
+ <p>Register new resource model</p>
852
+
853
+
854
+ </div>
855
+ </div>
856
+ <div class="tags">
857
+ <p class="tag_title">Parameters:</p>
858
+ <ul class="param">
859
+
860
+ <li>
861
+
862
+ <span class='name'>symbol</span>
863
+
864
+
865
+ <span class='type'>(<tt>Symbol</tt>)</span>
866
+
867
+
868
+
869
+ </li>
870
+
871
+ <li>
872
+
873
+ <span class='name'>model</span>
874
+
875
+
876
+ <span class='type'>(<tt>Class</tt>)</span>
877
+
878
+
879
+
880
+ </li>
881
+
882
+ </ul>
883
+
884
+
885
+ </div><table class="source_code">
886
+ <tr>
887
+ <td>
888
+ <pre class="lines">
889
+
890
+
891
+ 52
892
+ 53
893
+ 54
894
+ 55
895
+ 56
896
+ 57
897
+ 58
898
+ 59
899
+ 60
900
+ 61</pre>
901
+ </td>
902
+ <td>
903
+ <pre class="code"><span class="info file"># File 'lib/tango/app.rb', line 52</span>
904
+
905
+ <span class='kw'>def</span> <span class='id identifier rubyid_register_model'>register_model</span><span class='lparen'>(</span> <span class='id identifier rubyid_symbol'>symbol</span><span class='comma'>,</span> <span class='id identifier rubyid_model'>model</span> <span class='rparen'>)</span>
906
+
907
+ <span class='ivar'>@models</span><span class='lbracket'>[</span><span class='id identifier rubyid_symbol'>symbol</span><span class='rbracket'>]</span> <span class='op'>=</span> <span class='id identifier rubyid_model'>model</span>
908
+
909
+ <span class='comment'># Truncate table of non persistent model
910
+ </span> <span class='kw'>unless</span> <span class='id identifier rubyid_model'>model</span><span class='period'>.</span><span class='id identifier rubyid_persistent?'>persistent?</span>
911
+ <span class='const'>ActiveRecord</span><span class='op'>::</span><span class='const'>Base</span><span class='period'>.</span><span class='id identifier rubyid_connection'>connection</span><span class='period'>.</span><span class='id identifier rubyid_execute'>execute</span><span class='lparen'>(</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>TRUNCATE </span><span class='embexpr_beg'>#{</span><span class='id identifier rubyid_model'>model</span><span class='period'>.</span><span class='id identifier rubyid_table_name'>table_name</span><span class='embexpr_end'>}</span><span class='tstring_end'>&quot;</span></span> <span class='rparen'>)</span>
912
+ <span class='kw'>end</span>
913
+
914
+ <span class='kw'>end</span></pre>
915
+ </td>
916
+ </tr>
917
+ </table>
918
+ </div>
919
+
920
+ <div class="method_details ">
921
+ <h3 class="signature " id="register_operator-instance_method">
922
+
923
+ - (<tt>Object</tt>) <strong>register_operator</strong>(symbol, operator)
924
+
925
+
926
+
927
+
928
+
929
+ </h3><div class="docstring">
930
+ <div class="discussion">
931
+
932
+ <p>Register new resource operator</p>
933
+
934
+
935
+ </div>
936
+ </div>
937
+ <div class="tags">
938
+ <p class="tag_title">Parameters:</p>
939
+ <ul class="param">
940
+
941
+ <li>
942
+
943
+ <span class='name'>symbol</span>
944
+
945
+
946
+ <span class='type'>(<tt>Symbol</tt>)</span>
947
+
948
+
949
+
950
+ </li>
951
+
952
+ <li>
953
+
954
+ <span class='name'>operator</span>
955
+
956
+
957
+ <span class='type'>(<tt>Class</tt>)</span>
958
+
959
+
960
+
961
+ </li>
962
+
963
+ </ul>
964
+
965
+
966
+ </div><table class="source_code">
967
+ <tr>
968
+ <td>
969
+ <pre class="lines">
970
+
971
+
972
+ 67
973
+ 68
974
+ 69
975
+ 70
976
+ 71
977
+ 72
978
+ 73
979
+ 74
980
+ 75
981
+ 76</pre>
982
+ </td>
983
+ <td>
984
+ <pre class="code"><span class="info file"># File 'lib/tango/app.rb', line 67</span>
985
+
986
+ <span class='kw'>def</span> <span class='id identifier rubyid_register_operator'>register_operator</span><span class='lparen'>(</span> <span class='id identifier rubyid_symbol'>symbol</span><span class='comma'>,</span> <span class='id identifier rubyid_operator'>operator</span> <span class='rparen'>)</span>
987
+
988
+ <span class='ivar'>@operators</span><span class='lbracket'>[</span><span class='id identifier rubyid_symbol'>symbol</span><span class='rbracket'>]</span> <span class='op'>=</span> <span class='id identifier rubyid_operator'>operator</span>
989
+
990
+ <span class='comment'># Register operator with resource cache system
991
+ </span> <span class='ivar'>@cache</span><span class='period'>.</span><span class='id identifier rubyid_register'>register</span><span class='lparen'>(</span> <span class='id identifier rubyid_symbol'>symbol</span> <span class='rparen'>)</span> <span class='kw'>do</span> <span class='op'>|</span><span class='id identifier rubyid_resource'>resource</span><span class='op'>|</span>
992
+ <span class='id identifier rubyid_operator'>operator</span><span class='period'>.</span><span class='id identifier rubyid_load'>load</span><span class='lparen'>(</span> <span class='id identifier rubyid_resource'>resource</span> <span class='rparen'>)</span>
993
+ <span class='kw'>end</span>
994
+
995
+ <span class='kw'>end</span></pre>
996
+ </td>
997
+ </tr>
998
+ </table>
999
+ </div>
1000
+
1001
+ <div class="method_details ">
1002
+ <h3 class="signature " id="run-instance_method">
1003
+
1004
+ - (<tt>Integer</tt>) <strong>run</strong>
1005
+
1006
+
1007
+
1008
+
1009
+
1010
+ </h3><div class="docstring">
1011
+ <div class="discussion">
1012
+
1013
+ <p>Run ETL process</p>
1014
+
1015
+
1016
+ </div>
1017
+ </div>
1018
+ <div class="tags">
1019
+ <p class="tag_title">Parameters:</p>
1020
+ <ul class="param">
1021
+
1022
+ <li>
1023
+
1024
+ <span class='name'>link_stack</span>
1025
+
1026
+
1027
+ <span class='type'>(<tt><span class='object_link'><a href="LinkStack.html" title="Tango::LinkStack (class)">Tango::LinkStack</a></span></tt>)</span>
1028
+
1029
+
1030
+
1031
+ </li>
1032
+
1033
+ <li>
1034
+
1035
+ <span class='name'>dispatcher</span>
1036
+
1037
+
1038
+ <span class='type'>(<tt>Tango::Etl::Dispatcher</tt>)</span>
1039
+
1040
+
1041
+
1042
+ </li>
1043
+
1044
+ <li>
1045
+
1046
+ <span class='name'>cache</span>
1047
+
1048
+
1049
+ <span class='type'>(<tt>Tango::Resources::Cache</tt>)</span>
1050
+
1051
+
1052
+
1053
+ </li>
1054
+
1055
+ <li>
1056
+
1057
+ <span class='name'>http_client</span>
1058
+
1059
+
1060
+ <span class='type'>(<tt>Object</tt>)</span>
1061
+
1062
+
1063
+
1064
+ &mdash;
1065
+ <div class='inline'>
1066
+ <p>Must implement get method</p>
1067
+ </div>
1068
+
1069
+ </li>
1070
+
1071
+ <li>
1072
+
1073
+ <span class='name'>parser</span>
1074
+
1075
+
1076
+ <span class='type'>(<tt>Object</tt>)</span>
1077
+
1078
+
1079
+
1080
+ &mdash;
1081
+ <div class='inline'>
1082
+ <p>Must implement parse method</p>
1083
+ </div>
1084
+
1085
+ </li>
1086
+
1087
+ <li>
1088
+
1089
+ <span class='name'>logger</span>
1090
+
1091
+
1092
+ <span class='type'>(<tt>Logger</tt>)</span>
1093
+
1094
+
1095
+
1096
+ </li>
1097
+
1098
+ </ul>
1099
+
1100
+ <p class="tag_title">Returns:</p>
1101
+ <ul class="return">
1102
+
1103
+ <li>
1104
+
1105
+
1106
+ <span class='type'>(<tt>Integer</tt>)</span>
1107
+
1108
+
1109
+
1110
+ </li>
1111
+
1112
+ </ul>
1113
+
1114
+ </div><table class="source_code">
1115
+ <tr>
1116
+ <td>
1117
+ <pre class="lines">
1118
+
1119
+
1120
+ 87
1121
+ 88
1122
+ 89
1123
+ 90
1124
+ 91
1125
+ 92
1126
+ 93
1127
+ 94
1128
+ 95
1129
+ 96
1130
+ 97
1131
+ 98
1132
+ 99
1133
+ 100
1134
+ 101
1135
+ 102
1136
+ 103
1137
+ 104
1138
+ 105
1139
+ 106
1140
+ 107
1141
+ 108
1142
+ 109
1143
+ 110
1144
+ 111
1145
+ 112
1146
+ 113
1147
+ 114
1148
+ 115
1149
+ 116
1150
+ 117
1151
+ 118
1152
+ 119
1153
+ 120
1154
+ 121
1155
+ 122
1156
+ 123
1157
+ 124
1158
+ 125
1159
+ 126
1160
+ 127
1161
+ 128
1162
+ 129
1163
+ 130
1164
+ 131
1165
+ 132
1166
+ 133
1167
+ 134
1168
+ 135
1169
+ 136
1170
+ 137
1171
+ 138
1172
+ 139
1173
+ 140
1174
+ 141
1175
+ 142
1176
+ 143
1177
+ 144
1178
+ 145
1179
+ 146
1180
+ 147
1181
+ 148
1182
+ 149
1183
+ 150
1184
+ 151
1185
+ 152
1186
+ 153
1187
+ 154
1188
+ 155
1189
+ 156
1190
+ 157
1191
+ 158
1192
+ 159
1193
+ 160
1194
+ 161
1195
+ 162
1196
+ 163
1197
+ 164
1198
+ 165
1199
+ 166
1200
+ 167
1201
+ 168
1202
+ 169
1203
+ 170
1204
+ 171
1205
+ 172
1206
+ 173
1207
+ 174
1208
+ 175
1209
+ 176
1210
+ 177
1211
+ 178
1212
+ 179
1213
+ 180
1214
+ 181
1215
+ 182
1216
+ 183</pre>
1217
+ </td>
1218
+ <td>
1219
+ <pre class="code"><span class="info file"># File 'lib/tango/app.rb', line 87</span>
1220
+
1221
+ <span class='kw'>def</span> <span class='id identifier rubyid_run'>run</span>
1222
+
1223
+ <span class='comment'># Save beginning time
1224
+ </span> <span class='id identifier rubyid_start_time'>start_time</span> <span class='op'>=</span> <span class='const'>Time</span><span class='period'>.</span><span class='id identifier rubyid_now'>now</span>
1225
+
1226
+ <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_info'>info</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Running Tango v.</span><span class='embexpr_beg'>#{</span><span class='const'>Tango</span><span class='op'>::</span><span class='const'>VERSION</span><span class='embexpr_end'>}</span><span class='tstring_content'> ...</span><span class='tstring_end'>&quot;</span></span>
1227
+ <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_info'>info</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Target: </span><span class='embexpr_beg'>#{</span><span class='ivar'>@link_stack</span><span class='period'>.</span><span class='id identifier rubyid_host'>host</span><span class='embexpr_end'>}</span><span class='tstring_content'>.</span><span class='tstring_end'>&quot;</span></span>
1228
+
1229
+ <span class='comment'># Use next unlocked database
1230
+ </span> <span class='const'>Multidb</span><span class='period'>.</span><span class='id identifier rubyid_use'>use</span><span class='lparen'>(</span> <span class='ivar'>@db_locker</span><span class='period'>.</span><span class='id identifier rubyid_unlocked'>unlocked</span> <span class='rparen'>)</span>
1231
+ <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_info'>info</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Using database &#39;</span><span class='embexpr_beg'>#{</span><span class='ivar'>@db_locker</span><span class='period'>.</span><span class='id identifier rubyid_unlocked'>unlocked</span><span class='embexpr_end'>}</span><span class='tstring_content'>&#39;.</span><span class='tstring_end'>&quot;</span></span>
1232
+
1233
+ <span class='comment'># Run before filter
1234
+ </span> <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_info'>info</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Loading cache ...</span><span class='tstring_end'>&quot;</span></span>
1235
+ <span class='id identifier rubyid_load_cache'>load_cache</span>
1236
+
1237
+ <span class='comment'># Run before filter
1238
+ </span> <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_info'>info</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Running before callback ...</span><span class='tstring_end'>&quot;</span></span>
1239
+ <span class='id identifier rubyid_before'>before</span>
1240
+
1241
+ <span class='comment'># Init counter of crawled links
1242
+ </span> <span class='id identifier rubyid_links_counter'>links_counter</span> <span class='op'>=</span> <span class='int'>0</span>
1243
+ <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_info'>info</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Tango starts crawling ...</span><span class='tstring_end'>&quot;</span></span>
1244
+
1245
+ <span class='comment'># Start crawling website
1246
+ </span> <span class='kw'>while</span><span class='lparen'>(</span> <span class='ivar'>@link_stack</span><span class='period'>.</span><span class='id identifier rubyid_has_links?'>has_links?</span> <span class='rparen'>)</span>
1247
+
1248
+ <span class='comment'># Get a link from the stack
1249
+ </span> <span class='id identifier rubyid_link'>link</span> <span class='op'>=</span> <span class='ivar'>@link_stack</span><span class='period'>.</span><span class='id identifier rubyid_shift'>shift</span>
1250
+
1251
+ <span class='comment'># Skip iteration if no handler found
1252
+ </span> <span class='kw'>if</span> <span class='op'>!</span> <span class='id identifier rubyid_handler_klass'>handler_klass</span> <span class='op'>=</span> <span class='ivar'>@dispatcher</span><span class='period'>.</span><span class='id identifier rubyid_find_handler'>find_handler</span><span class='lparen'>(</span> <span class='id identifier rubyid_link'>link</span> <span class='rparen'>)</span>
1253
+ <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_error'>error</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>No handler for link: </span><span class='embexpr_beg'>#{</span><span class='id identifier rubyid_link'>link</span><span class='embexpr_end'>}</span><span class='tstring_content'>.</span><span class='tstring_end'>&quot;</span></span>
1254
+ <span class='kw'>next</span>
1255
+ <span class='kw'>end</span>
1256
+
1257
+ <span class='comment'># Try to get contents of the link
1258
+ </span> <span class='kw'>begin</span>
1259
+ <span class='id identifier rubyid_response'>response</span> <span class='op'>=</span> <span class='ivar'>@http_client</span><span class='period'>.</span><span class='id identifier rubyid_get'>get</span><span class='lparen'>(</span> <span class='ivar'>@link_stack</span><span class='period'>.</span><span class='id identifier rubyid_host'>host</span> <span class='op'>+</span> <span class='id identifier rubyid_link'>link</span> <span class='rparen'>)</span>
1260
+ <span class='kw'>rescue</span> <span class='const'>StandardError</span> <span class='op'>=&gt;</span> <span class='id identifier rubyid_e'>e</span>
1261
+ <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_error'>error</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Could not download contents of </span><span class='embexpr_beg'>#{</span><span class='ivar'>@link_stack</span><span class='period'>.</span><span class='id identifier rubyid_host'>host</span> <span class='op'>+</span> <span class='id identifier rubyid_link'>link</span><span class='embexpr_end'>}</span><span class='tstring_content'> link.</span><span class='tstring_end'>&quot;</span></span><span class='semicolon'>;</span> <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_error'>error</span> <span class='id identifier rubyid_e'>e</span><span class='period'>.</span><span class='id identifier rubyid_message'>message</span>
1262
+ <span class='kw'>next</span>
1263
+ <span class='kw'>end</span>
1264
+
1265
+ <span class='comment'># Continue only when response has code 200 or 201
1266
+ </span> <span class='kw'>if</span> <span class='op'>!</span> <span class='lbracket'>[</span> <span class='int'>200</span><span class='comma'>,</span> <span class='int'>201</span> <span class='rbracket'>]</span><span class='period'>.</span><span class='id identifier rubyid_include?'>include?</span><span class='lparen'>(</span> <span class='id identifier rubyid_response'>response</span><span class='period'>.</span><span class='id identifier rubyid_code'>code</span> <span class='rparen'>)</span>
1267
+ <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_error'>error</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Response code for link </span><span class='embexpr_beg'>#{</span><span class='id identifier rubyid_link'>link</span><span class='embexpr_end'>}</span><span class='tstring_content'> is </span><span class='embexpr_beg'>#{</span><span class='id identifier rubyid_response'>response</span><span class='period'>.</span><span class='id identifier rubyid_code'>code</span><span class='embexpr_end'>}</span><span class='tstring_content'>. Only code 200 is accepted.</span><span class='tstring_end'>&quot;</span></span>
1268
+ <span class='kw'>next</span>
1269
+ <span class='kw'>end</span>
1270
+
1271
+ <span class='comment'># Parse response contents
1272
+ </span> <span class='id identifier rubyid_document'>document</span> <span class='op'>=</span> <span class='ivar'>@parser</span><span class='period'>.</span><span class='id identifier rubyid_parse'>parse</span><span class='lparen'>(</span> <span class='id identifier rubyid_response'>response</span><span class='period'>.</span><span class='id identifier rubyid_body'>body</span> <span class='rparen'>)</span>
1273
+ <span class='comment'># Init handler
1274
+ </span> <span class='id identifier rubyid_handler'>handler</span> <span class='op'>=</span> <span class='id identifier rubyid_handler_klass'>handler_klass</span><span class='period'>.</span><span class='id identifier rubyid_new'>new</span><span class='lparen'>(</span> <span class='id identifier rubyid_link'>link</span><span class='comma'>,</span> <span class='id identifier rubyid_document'>document</span><span class='comma'>,</span> <span class='ivar'>@cache</span> <span class='rparen'>)</span>
1275
+
1276
+ <span class='comment'># Append links fetched from handler
1277
+ </span> <span class='ivar'>@link_stack</span><span class='period'>.</span><span class='id identifier rubyid_append'>append</span><span class='lparen'>(</span> <span class='id identifier rubyid_handler'>handler</span><span class='period'>.</span><span class='id identifier rubyid_links'>links</span> <span class='rparen'>)</span>
1278
+
1279
+ <span class='comment'># Try to fire the handler
1280
+ </span> <span class='kw'>begin</span>
1281
+ <span class='id identifier rubyid_handler'>handler</span><span class='period'>.</span><span class='id identifier rubyid_trigger'>trigger</span>
1282
+ <span class='kw'>rescue</span> <span class='const'>StandardError</span> <span class='op'>=&gt;</span> <span class='id identifier rubyid_e'>e</span>
1283
+ <span class='comment'># Log error
1284
+ </span> <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_error'>error</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Link: </span><span class='embexpr_beg'>#{</span><span class='id identifier rubyid_link'>link</span><span class='embexpr_end'>}</span><span class='tstring_content'>. Handler had some troubles.</span><span class='tstring_end'>&quot;</span></span>
1285
+ <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_error'>error</span> <span class='id identifier rubyid_e'>e</span><span class='period'>.</span><span class='id identifier rubyid_message'>message</span>
1286
+ <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_error'>error</span> <span class='id identifier rubyid_e'>e</span><span class='period'>.</span><span class='id identifier rubyid_backtrace'>backtrace</span><span class='period'>.</span><span class='id identifier rubyid_join'>join</span><span class='lparen'>(</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>\n</span><span class='tstring_end'>&quot;</span></span> <span class='rparen'>)</span>
1287
+ <span class='kw'>else</span>
1288
+ <span class='id identifier rubyid_links_counter'>links_counter</span> <span class='op'>+=</span> <span class='int'>1</span>
1289
+ <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_debug'>debug</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Link: </span><span class='embexpr_beg'>#{</span><span class='id identifier rubyid_link'>link</span><span class='embexpr_end'>}</span><span class='tstring_content'>. Handler triggered successfully.</span><span class='tstring_end'>&quot;</span></span>
1290
+ <span class='kw'>end</span>
1291
+
1292
+ <span class='comment'># Sleep to give crawled server time to breath
1293
+ </span> <span class='id identifier rubyid_sleep'>sleep</span><span class='lparen'>(</span> <span class='ivar'>@config</span><span class='lbracket'>[</span><span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>sleep</span><span class='tstring_end'>&quot;</span></span><span class='rbracket'>]</span> <span class='op'>||</span> <span class='int'>0</span> <span class='rparen'>)</span>
1294
+
1295
+ <span class='kw'>end</span>
1296
+
1297
+ <span class='comment'># Release buffers
1298
+ </span> <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_info'>info</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Releasing buffers ...</span><span class='tstring_end'>&quot;</span></span>
1299
+ <span class='ivar'>@cache</span><span class='period'>.</span><span class='id identifier rubyid_buffer'>buffer</span><span class='period'>.</span><span class='id identifier rubyid_release_all'>release_all</span><span class='lparen'>(</span><span class='rparen'>)</span>
1300
+
1301
+ <span class='comment'># Run after filter
1302
+ </span> <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_info'>info</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Running after callback ...</span><span class='tstring_end'>&quot;</span></span>
1303
+ <span class='id identifier rubyid_after'>after</span>
1304
+
1305
+ <span class='comment'># Lock database used in this Tango iteration
1306
+ </span> <span class='ivar'>@db_locker</span><span class='period'>.</span><span class='id identifier rubyid_lock'>lock</span><span class='lparen'>(</span> <span class='ivar'>@db_locker</span><span class='period'>.</span><span class='id identifier rubyid_unlocked'>unlocked</span> <span class='rparen'>)</span>
1307
+
1308
+ <span class='comment'># Get time of script execution ending
1309
+ </span> <span class='id identifier rubyid_end_time'>end_time</span> <span class='op'>=</span> <span class='const'>Time</span><span class='period'>.</span><span class='id identifier rubyid_now'>now</span>
1310
+
1311
+ <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_info'>info</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Tango crawled </span><span class='embexpr_beg'>#{</span><span class='id identifier rubyid_links_counter'>links_counter</span><span class='embexpr_end'>}</span><span class='tstring_content'>/</span><span class='embexpr_beg'>#{</span><span class='ivar'>@link_stack</span><span class='period'>.</span><span class='id identifier rubyid_shifted'>shifted</span><span class='embexpr_end'>}</span><span class='tstring_content'> links successfully.</span><span class='tstring_end'>&quot;</span></span>
1312
+ <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_info'>info</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Start time: </span><span class='embexpr_beg'>#{</span><span class='id identifier rubyid_start_time'>start_time</span><span class='embexpr_end'>}</span><span class='tstring_content'>, end time: </span><span class='embexpr_beg'>#{</span><span class='id identifier rubyid_end_time'>end_time</span><span class='embexpr_end'>}</span><span class='tstring_content'>, time elapsed: </span><span class='embexpr_beg'>#{</span><span class='id identifier rubyid_end_time'>end_time</span> <span class='op'>-</span> <span class='id identifier rubyid_start_time'>start_time</span><span class='embexpr_end'>}</span><span class='tstring_content'> seconds.</span><span class='tstring_end'>&quot;</span></span>
1313
+
1314
+ <span class='comment'># Close logger
1315
+ </span> <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_close'>close</span>
1316
+
1317
+ <span class='kw'>end</span></pre>
1318
+ </td>
1319
+ </tr>
1320
+ </table>
1321
+ </div>
1322
+
1323
+ </div>
1324
+
1325
+ </div>
1326
+
1327
+ <div id="footer">
1328
+ Generated on Tue Apr 15 02:47:42 2014 by
1329
+ <a href="http://yardoc.org" title="Yay! A Ruby Documentation Tool" target="_parent">yard</a>
1330
+ 0.8.7.4 (ruby-2.0.0).
1331
+ </div>
1332
+
1333
+ </body>
1334
+ </html>