tango-etl 0.0.1 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +4 -4
  2. data/{changelog.md → CHANGELOG.md} +5 -1
  3. data/{readme.md → README.md} +1 -1
  4. data/doc/Multidb/Balancer.html +202 -0
  5. data/doc/Multidb.html +184 -0
  6. data/doc/Tango/AbstractModel.html +608 -0
  7. data/doc/Tango/App.html +1334 -0
  8. data/doc/Tango/DatabaseLocker.html +541 -0
  9. data/doc/Tango/ETL/Dispatcher.html +465 -0
  10. data/doc/Tango/ETL/HandlerInterface.html +639 -0
  11. data/doc/Tango/ETL/OperatorInterface.html +494 -0
  12. data/doc/Tango/ETL.html +117 -0
  13. data/doc/Tango/Kernel.html +432 -0
  14. data/doc/Tango/LinkStack.html +737 -0
  15. data/doc/Tango/Resource/Buffer.html +565 -0
  16. data/doc/Tango/Resource/Cache.html +834 -0
  17. data/doc/Tango/Resource.html +117 -0
  18. data/doc/Tango.html +131 -0
  19. data/doc/_index.html +297 -0
  20. data/doc/class_list.html +54 -0
  21. data/doc/css/common.css +1 -0
  22. data/doc/css/full_list.css +57 -0
  23. data/doc/css/style.css +339 -0
  24. data/doc/file.README.html +80 -0
  25. data/doc/file_list.html +56 -0
  26. data/doc/frames.html +26 -0
  27. data/doc/index.html +80 -0
  28. data/doc/js/app.js +219 -0
  29. data/doc/js/full_list.js +178 -0
  30. data/doc/js/jquery.js +4 -0
  31. data/doc/method_list.html +365 -0
  32. data/doc/top-level-namespace.html +112 -0
  33. data/lib/tango/abstract_model.rb +1 -0
  34. data/lib/tango/app.rb +4 -7
  35. data/lib/tango/database_locker.rb +23 -2
  36. data/lib/tango/etl/handler_interface.rb +12 -6
  37. data/lib/tango/resource/buffer.rb +7 -7
  38. data/lib/tango/resource/cache.rb +18 -13
  39. data/lib/tango/version.rb +1 -1
  40. data/test/support/lib/simple_handler.rb +5 -14
  41. data/test/unit/etl/test_dispatcher.rb +21 -5
  42. metadata +33 -6
  43. data/config/app.yml.sample +0 -6
  44. data/config/database.yml.sample +0 -27
data/doc/Tango/App.html ADDED
@@ -0,0 +1,1334 @@
1
+ <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
2
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
3
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
4
+ <head>
5
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
6
+ <title>
7
+ Class: Tango::App
8
+
9
+ &mdash; Documentation by YARD 0.8.7.4
10
+
11
+ </title>
12
+
13
+ <link rel="stylesheet" href="../css/style.css" type="text/css" charset="utf-8" />
14
+
15
+ <link rel="stylesheet" href="../css/common.css" type="text/css" charset="utf-8" />
16
+
17
+ <script type="text/javascript" charset="utf-8">
18
+ hasFrames = window.top.frames.main ? true : false;
19
+ relpath = '../';
20
+ framesUrl = "../frames.html#!Tango/App.html";
21
+ </script>
22
+
23
+
24
+ <script type="text/javascript" charset="utf-8" src="../js/jquery.js"></script>
25
+
26
+ <script type="text/javascript" charset="utf-8" src="../js/app.js"></script>
27
+
28
+
29
+ </head>
30
+ <body>
31
+ <div id="header">
32
+ <div id="menu">
33
+
34
+ <a href="../_index.html">Index (A)</a> &raquo;
35
+ <span class='title'><span class='object_link'><a href="../Tango.html" title="Tango (module)">Tango</a></span></span>
36
+ &raquo;
37
+ <span class="title">App</span>
38
+
39
+
40
+ <div class="noframes"><span class="title">(</span><a href="." target="_top">no frames</a><span class="title">)</span></div>
41
+ </div>
42
+
43
+ <div id="search">
44
+
45
+ <a class="full_list_link" id="class_list_link"
46
+ href="../class_list.html">
47
+ Class List
48
+ </a>
49
+
50
+ <a class="full_list_link" id="method_list_link"
51
+ href="../method_list.html">
52
+ Method List
53
+ </a>
54
+
55
+ <a class="full_list_link" id="file_list_link"
56
+ href="../file_list.html">
57
+ File List
58
+ </a>
59
+
60
+ </div>
61
+ <div class="clear"></div>
62
+ </div>
63
+
64
+ <iframe id="search_frame"></iframe>
65
+
66
+ <div id="content"><h1>Class: Tango::App
67
+
68
+
69
+
70
+ </h1>
71
+
72
+ <dl class="box">
73
+
74
+ <dt class="r1">Inherits:</dt>
75
+ <dd class="r1">
76
+ <span class="inheritName">Object</span>
77
+
78
+ <ul class="fullTree">
79
+ <li>Object</li>
80
+
81
+ <li class="next">Tango::App</li>
82
+
83
+ </ul>
84
+ <a href="#" class="inheritanceTree">show all</a>
85
+
86
+ </dd>
87
+
88
+
89
+
90
+
91
+
92
+
93
+
94
+
95
+
96
+ <dt class="r2 last">Defined in:</dt>
97
+ <dd class="r2 last">lib/tango/app.rb</dd>
98
+
99
+ </dl>
100
+ <div class="clear"></div>
101
+
102
+ <h2>Overview</h2><div class="docstring">
103
+ <div class="discussion">
104
+
105
+ <p>Tango application</p>
106
+
107
+
108
+ </div>
109
+ </div>
110
+ <div class="tags">
111
+
112
+ <p class="tag_title">Author:</p>
113
+ <ul class="author">
114
+
115
+ <li>
116
+
117
+
118
+
119
+
120
+
121
+ <div class='inline'>
122
+ <p>Mckomo</p>
123
+ </div>
124
+
125
+ </li>
126
+
127
+ </ul>
128
+
129
+ </div>
130
+
131
+
132
+
133
+ <h2>Instance Attribute Summary <small>(<a href="#" class="summary_toggle">collapse</a>)</small></h2>
134
+ <ul class="summary">
135
+
136
+ <li class="public ">
137
+ <span class="summary_signature">
138
+
139
+ <a href="#config-instance_method" title="#config (instance method)">- (Object) <strong>config</strong> </a>
140
+
141
+
142
+
143
+ </span>
144
+
145
+
146
+
147
+
148
+ <span class="note title readonly">readonly</span>
149
+
150
+
151
+
152
+
153
+
154
+
155
+
156
+
157
+
158
+ <span class="summary_desc"><div class='inline'>
159
+ <p>Returns the value of attribute config.</p>
160
+ </div></span>
161
+
162
+ </li>
163
+
164
+
165
+ <li class="public ">
166
+ <span class="summary_signature">
167
+
168
+ <a href="#dispatcher-instance_method" title="#dispatcher (instance method)">- (Object) <strong>dispatcher</strong> </a>
169
+
170
+
171
+
172
+ </span>
173
+
174
+
175
+
176
+
177
+ <span class="note title readonly">readonly</span>
178
+
179
+
180
+
181
+
182
+
183
+
184
+
185
+
186
+
187
+ <span class="summary_desc"><div class='inline'>
188
+ <p>Returns the value of attribute dispatcher.</p>
189
+ </div></span>
190
+
191
+ </li>
192
+
193
+
194
+ <li class="public ">
195
+ <span class="summary_signature">
196
+
197
+ <a href="#link_stack-instance_method" title="#link_stack (instance method)">- (Object) <strong>link_stack</strong> </a>
198
+
199
+
200
+
201
+ </span>
202
+
203
+
204
+
205
+
206
+ <span class="note title readonly">readonly</span>
207
+
208
+
209
+
210
+
211
+
212
+
213
+
214
+
215
+
216
+ <span class="summary_desc"><div class='inline'>
217
+ <p>Returns the value of attribute link_stack.</p>
218
+ </div></span>
219
+
220
+ </li>
221
+
222
+
223
+ <li class="public ">
224
+ <span class="summary_signature">
225
+
226
+ <a href="#logger-instance_method" title="#logger (instance method)">- (Object) <strong>logger</strong> </a>
227
+
228
+
229
+
230
+ </span>
231
+
232
+
233
+
234
+
235
+ <span class="note title readonly">readonly</span>
236
+
237
+
238
+
239
+
240
+
241
+
242
+
243
+
244
+
245
+ <span class="summary_desc"><div class='inline'>
246
+ <p>Returns the value of attribute logger.</p>
247
+ </div></span>
248
+
249
+ </li>
250
+
251
+
252
+ </ul>
253
+
254
+
255
+
256
+
257
+
258
+ <h2>
259
+ Instance Method Summary
260
+ <small>(<a href="#" class="summary_toggle">collapse</a>)</small>
261
+ </h2>
262
+
263
+ <ul class="summary">
264
+
265
+ <li class="public ">
266
+ <span class="summary_signature">
267
+
268
+ <a href="#after-instance_method" title="#after (instance method)">- (Object) <strong>after</strong> </a>
269
+
270
+
271
+
272
+ </span>
273
+
274
+
275
+
276
+
277
+
278
+
279
+
280
+
281
+
282
+ <span class="summary_desc"><div class='inline'>
283
+ <p>Filter run after Tango execution.</p>
284
+ </div></span>
285
+
286
+ </li>
287
+
288
+
289
+ <li class="public ">
290
+ <span class="summary_signature">
291
+
292
+ <a href="#before-instance_method" title="#before (instance method)">- (Object) <strong>before</strong> </a>
293
+
294
+
295
+
296
+ </span>
297
+
298
+
299
+
300
+
301
+
302
+
303
+
304
+
305
+
306
+ <span class="summary_desc"><div class='inline'>
307
+ <p>Filter run before Tango execution.</p>
308
+ </div></span>
309
+
310
+ </li>
311
+
312
+
313
+ <li class="public ">
314
+ <span class="summary_signature">
315
+
316
+ <a href="#initialize-instance_method" title="#initialize (instance method)">- (Tango::App) <strong>initialize</strong>(config: {}, link_stack: nil, dispatcher: nil, cache: nil, http_client: nil, parser: nil, db_locker: nil, logger: nil) </a>
317
+
318
+
319
+
320
+ </span>
321
+
322
+
323
+ <span class="note title constructor">constructor</span>
324
+
325
+
326
+
327
+
328
+
329
+
330
+
331
+
332
+ <span class="summary_desc"><div class='inline'></div></span>
333
+
334
+ </li>
335
+
336
+
337
+ <li class="public ">
338
+ <span class="summary_signature">
339
+
340
+ <a href="#register_model-instance_method" title="#register_model (instance method)">- (Object) <strong>register_model</strong>(symbol, model) </a>
341
+
342
+
343
+
344
+ </span>
345
+
346
+
347
+
348
+
349
+
350
+
351
+
352
+
353
+
354
+ <span class="summary_desc"><div class='inline'>
355
+ <p>Register new resource model.</p>
356
+ </div></span>
357
+
358
+ </li>
359
+
360
+
361
+ <li class="public ">
362
+ <span class="summary_signature">
363
+
364
+ <a href="#register_operator-instance_method" title="#register_operator (instance method)">- (Object) <strong>register_operator</strong>(symbol, operator) </a>
365
+
366
+
367
+
368
+ </span>
369
+
370
+
371
+
372
+
373
+
374
+
375
+
376
+
377
+
378
+ <span class="summary_desc"><div class='inline'>
379
+ <p>Register new resource operator.</p>
380
+ </div></span>
381
+
382
+ </li>
383
+
384
+
385
+ <li class="public ">
386
+ <span class="summary_signature">
387
+
388
+ <a href="#run-instance_method" title="#run (instance method)">- (Integer) <strong>run</strong> </a>
389
+
390
+
391
+
392
+ </span>
393
+
394
+
395
+
396
+
397
+
398
+
399
+
400
+
401
+
402
+ <span class="summary_desc"><div class='inline'>
403
+ <p>Run ETL process.</p>
404
+ </div></span>
405
+
406
+ </li>
407
+
408
+
409
+ </ul>
410
+
411
+
412
+ <div id="constructor_details" class="method_details_list">
413
+ <h2>Constructor Details</h2>
414
+
415
+ <div class="method_details first">
416
+ <h3 class="signature first" id="initialize-instance_method">
417
+
418
+ - (<tt><span class='object_link'><a href="" title="Tango::App (class)">Tango::App</a></span></tt>) <strong>initialize</strong>(config: {}, link_stack: nil, dispatcher: nil, cache: nil, http_client: nil, parser: nil, db_locker: nil, logger: nil)
419
+
420
+
421
+
422
+
423
+
424
+ </h3><div class="docstring">
425
+ <div class="discussion">
426
+
427
+
428
+ </div>
429
+ </div>
430
+ <div class="tags">
431
+ <p class="tag_title">Parameters:</p>
432
+ <ul class="param">
433
+
434
+ <li>
435
+
436
+ <span class='name'>link_stack</span>
437
+
438
+
439
+ <span class='type'>(<tt><span class='object_link'><a href="LinkStack.html" title="Tango::LinkStack (class)">Tango::LinkStack</a></span></tt>)</span>
440
+
441
+
442
+
443
+ </li>
444
+
445
+ <li>
446
+
447
+ <span class='name'>dispatcher</span>
448
+
449
+
450
+ <span class='type'>(<tt>Tango::Etl::Dispatcher</tt>)</span>
451
+
452
+
453
+
454
+ </li>
455
+
456
+ <li>
457
+
458
+ <span class='name'>cache</span>
459
+
460
+
461
+ <span class='type'>(<tt>Tango::Resources::Cache</tt>)</span>
462
+
463
+
464
+
465
+ </li>
466
+
467
+ <li>
468
+
469
+ <span class='name'>http_client</span>
470
+
471
+
472
+ <span class='type'>(<tt>Object</tt>)</span>
473
+
474
+
475
+
476
+ &mdash;
477
+ <div class='inline'>
478
+ <p>Must implement get method</p>
479
+ </div>
480
+
481
+ </li>
482
+
483
+ <li>
484
+
485
+ <span class='name'>parser</span>
486
+
487
+
488
+ <span class='type'>(<tt>Object</tt>)</span>
489
+
490
+
491
+
492
+ &mdash;
493
+ <div class='inline'>
494
+ <p>Must implement parse method</p>
495
+ </div>
496
+
497
+ </li>
498
+
499
+ <li>
500
+
501
+ <span class='name'>db_locker</span>
502
+
503
+
504
+ <span class='type'>(<tt><span class='object_link'><a href="DatabaseLocker.html" title="Tango::DatabaseLocker (class)">DatabaseLocker</a></span></tt>)</span>
505
+
506
+
507
+
508
+ </li>
509
+
510
+ <li>
511
+
512
+ <span class='name'>logger</span>
513
+
514
+
515
+ <span class='type'>(<tt>Logger</tt>)</span>
516
+
517
+
518
+
519
+ </li>
520
+
521
+ </ul>
522
+
523
+
524
+ </div><table class="source_code">
525
+ <tr>
526
+ <td>
527
+ <pre class="lines">
528
+
529
+
530
+ 20
531
+ 21
532
+ 22
533
+ 23
534
+ 24
535
+ 25
536
+ 26
537
+ 27
538
+ 28
539
+ 29
540
+ 30
541
+ 31
542
+ 32
543
+ 33
544
+ 34
545
+ 35
546
+ 36
547
+ 37
548
+ 38</pre>
549
+ </td>
550
+ <td>
551
+ <pre class="code"><span class="info file"># File 'lib/tango/app.rb', line 20</span>
552
+
553
+ <span class='kw'>def</span> <span class='id identifier rubyid_initialize'>initialize</span><span class='lparen'>(</span> <span class='label'>config:</span> <span class='lbrace'>{</span><span class='rbrace'>}</span><span class='comma'>,</span> <span class='label'>link_stack:</span> <span class='kw'>nil</span><span class='comma'>,</span> <span class='label'>dispatcher:</span> <span class='kw'>nil</span><span class='comma'>,</span> <span class='label'>cache:</span> <span class='kw'>nil</span><span class='comma'>,</span> <span class='label'>http_client:</span> <span class='kw'>nil</span><span class='comma'>,</span> <span class='label'>parser:</span> <span class='kw'>nil</span><span class='comma'>,</span> <span class='label'>db_locker:</span> <span class='kw'>nil</span><span class='comma'>,</span> <span class='label'>logger:</span> <span class='kw'>nil</span> <span class='rparen'>)</span>
554
+
555
+ <span class='comment'># Init app properties
556
+ </span> <span class='ivar'>@models</span> <span class='op'>=</span> <span class='lbrace'>{</span><span class='rbrace'>}</span>
557
+ <span class='ivar'>@operators</span> <span class='op'>=</span> <span class='lbrace'>{</span><span class='rbrace'>}</span>
558
+
559
+ <span class='comment'># Set config
560
+ </span> <span class='ivar'>@config</span> <span class='op'>=</span> <span class='id identifier rubyid_config'>config</span>
561
+
562
+ <span class='comment'># Set dependencies
563
+ </span> <span class='ivar'>@link_stack</span> <span class='op'>=</span> <span class='id identifier rubyid_link_stack'>link_stack</span> <span class='op'>||</span> <span class='const'>LinkStack</span><span class='period'>.</span><span class='id identifier rubyid_new'>new</span><span class='lparen'>(</span> <span class='id identifier rubyid_config'>config</span><span class='lbracket'>[</span><span class='tstring'><span class='tstring_beg'>&#39;</span><span class='tstring_content'>target_url</span><span class='tstring_end'>&#39;</span></span><span class='rbracket'>]</span> <span class='rparen'>)</span>
564
+ <span class='ivar'>@dispatcher</span> <span class='op'>=</span> <span class='id identifier rubyid_dispatcher'>dispatcher</span> <span class='op'>||</span> <span class='const'>ETL</span><span class='op'>::</span><span class='const'>Dispatcher</span><span class='period'>.</span><span class='id identifier rubyid_new'>new</span>
565
+ <span class='ivar'>@cache</span> <span class='op'>=</span> <span class='id identifier rubyid_cache'>cache</span> <span class='op'>||</span> <span class='const'>Resource</span><span class='op'>::</span><span class='const'>Cache</span><span class='period'>.</span><span class='id identifier rubyid_new'>new</span><span class='lparen'>(</span> <span class='const'>Resource</span><span class='op'>::</span><span class='const'>Buffer</span><span class='period'>.</span><span class='id identifier rubyid_new'>new</span> <span class='rparen'>)</span>
566
+ <span class='ivar'>@http_client</span> <span class='op'>=</span> <span class='id identifier rubyid_http_client'>http_client</span> <span class='op'>||</span> <span class='const'>HTTParty</span>
567
+ <span class='ivar'>@parser</span> <span class='op'>=</span> <span class='id identifier rubyid_parser'>parser</span> <span class='op'>||</span> <span class='const'>Nokogiri</span><span class='op'>::</span><span class='const'>HTML</span>
568
+ <span class='ivar'>@db_locker</span> <span class='op'>=</span> <span class='id identifier rubyid_db_locker'>db_locker</span> <span class='op'>||</span> <span class='const'>DatabaseLocker</span><span class='period'>.</span><span class='id identifier rubyid_new'>new</span><span class='lparen'>(</span> <span class='const'>Multidb</span><span class='period'>.</span><span class='id identifier rubyid_databases'>databases</span> <span class='rparen'>)</span>
569
+ <span class='ivar'>@logger</span> <span class='op'>=</span> <span class='id identifier rubyid_logger'>logger</span> <span class='op'>||</span> <span class='const'>Logger</span><span class='period'>.</span><span class='id identifier rubyid_new'>new</span><span class='lparen'>(</span> <span class='const'>STDOUT</span> <span class='rparen'>)</span>
570
+
571
+ <span class='kw'>end</span></pre>
572
+ </td>
573
+ </tr>
574
+ </table>
575
+ </div>
576
+
577
+ </div>
578
+
579
+ <div id="instance_attr_details" class="attr_details">
580
+ <h2>Instance Attribute Details</h2>
581
+
582
+
583
+ <span id=""></span>
584
+ <div class="method_details first">
585
+ <h3 class="signature first" id="config-instance_method">
586
+
587
+ - (<tt>Object</tt>) <strong>config</strong> <span class="extras">(readonly)</span>
588
+
589
+
590
+
591
+
592
+
593
+ </h3><div class="docstring">
594
+ <div class="discussion">
595
+
596
+ <p>Returns the value of attribute config</p>
597
+
598
+
599
+ </div>
600
+ </div>
601
+ <div class="tags">
602
+
603
+
604
+ </div><table class="source_code">
605
+ <tr>
606
+ <td>
607
+ <pre class="lines">
608
+
609
+
610
+ 10
611
+ 11
612
+ 12</pre>
613
+ </td>
614
+ <td>
615
+ <pre class="code"><span class="info file"># File 'lib/tango/app.rb', line 10</span>
616
+
617
+ <span class='kw'>def</span> <span class='id identifier rubyid_config'>config</span>
618
+ <span class='ivar'>@config</span>
619
+ <span class='kw'>end</span></pre>
620
+ </td>
621
+ </tr>
622
+ </table>
623
+ </div>
624
+
625
+
626
+ <span id=""></span>
627
+ <div class="method_details ">
628
+ <h3 class="signature " id="dispatcher-instance_method">
629
+
630
+ - (<tt>Object</tt>) <strong>dispatcher</strong> <span class="extras">(readonly)</span>
631
+
632
+
633
+
634
+
635
+
636
+ </h3><div class="docstring">
637
+ <div class="discussion">
638
+
639
+ <p>Returns the value of attribute dispatcher</p>
640
+
641
+
642
+ </div>
643
+ </div>
644
+ <div class="tags">
645
+
646
+
647
+ </div><table class="source_code">
648
+ <tr>
649
+ <td>
650
+ <pre class="lines">
651
+
652
+
653
+ 10
654
+ 11
655
+ 12</pre>
656
+ </td>
657
+ <td>
658
+ <pre class="code"><span class="info file"># File 'lib/tango/app.rb', line 10</span>
659
+
660
+ <span class='kw'>def</span> <span class='id identifier rubyid_dispatcher'>dispatcher</span>
661
+ <span class='ivar'>@dispatcher</span>
662
+ <span class='kw'>end</span></pre>
663
+ </td>
664
+ </tr>
665
+ </table>
666
+ </div>
667
+
668
+
669
+ <span id=""></span>
670
+ <div class="method_details ">
671
+ <h3 class="signature " id="link_stack-instance_method">
672
+
673
+ - (<tt>Object</tt>) <strong>link_stack</strong> <span class="extras">(readonly)</span>
674
+
675
+
676
+
677
+
678
+
679
+ </h3><div class="docstring">
680
+ <div class="discussion">
681
+
682
+ <p>Returns the value of attribute link_stack</p>
683
+
684
+
685
+ </div>
686
+ </div>
687
+ <div class="tags">
688
+
689
+
690
+ </div><table class="source_code">
691
+ <tr>
692
+ <td>
693
+ <pre class="lines">
694
+
695
+
696
+ 10
697
+ 11
698
+ 12</pre>
699
+ </td>
700
+ <td>
701
+ <pre class="code"><span class="info file"># File 'lib/tango/app.rb', line 10</span>
702
+
703
+ <span class='kw'>def</span> <span class='id identifier rubyid_link_stack'>link_stack</span>
704
+ <span class='ivar'>@link_stack</span>
705
+ <span class='kw'>end</span></pre>
706
+ </td>
707
+ </tr>
708
+ </table>
709
+ </div>
710
+
711
+
712
+ <span id=""></span>
713
+ <div class="method_details ">
714
+ <h3 class="signature " id="logger-instance_method">
715
+
716
+ - (<tt>Object</tt>) <strong>logger</strong> <span class="extras">(readonly)</span>
717
+
718
+
719
+
720
+
721
+
722
+ </h3><div class="docstring">
723
+ <div class="discussion">
724
+
725
+ <p>Returns the value of attribute logger</p>
726
+
727
+
728
+ </div>
729
+ </div>
730
+ <div class="tags">
731
+
732
+
733
+ </div><table class="source_code">
734
+ <tr>
735
+ <td>
736
+ <pre class="lines">
737
+
738
+
739
+ 10
740
+ 11
741
+ 12</pre>
742
+ </td>
743
+ <td>
744
+ <pre class="code"><span class="info file"># File 'lib/tango/app.rb', line 10</span>
745
+
746
+ <span class='kw'>def</span> <span class='id identifier rubyid_logger'>logger</span>
747
+ <span class='ivar'>@logger</span>
748
+ <span class='kw'>end</span></pre>
749
+ </td>
750
+ </tr>
751
+ </table>
752
+ </div>
753
+
754
+ </div>
755
+
756
+
757
+ <div id="instance_method_details" class="method_details_list">
758
+ <h2>Instance Method Details</h2>
759
+
760
+
761
+ <div class="method_details first">
762
+ <h3 class="signature first" id="after-instance_method">
763
+
764
+ - (<tt>Object</tt>) <strong>after</strong>
765
+
766
+
767
+
768
+
769
+
770
+ </h3><div class="docstring">
771
+ <div class="discussion">
772
+
773
+ <p>Filter run after Tango execution</p>
774
+
775
+
776
+ </div>
777
+ </div>
778
+ <div class="tags">
779
+
780
+
781
+ </div><table class="source_code">
782
+ <tr>
783
+ <td>
784
+ <pre class="lines">
785
+
786
+
787
+ 45
788
+ 46</pre>
789
+ </td>
790
+ <td>
791
+ <pre class="code"><span class="info file"># File 'lib/tango/app.rb', line 45</span>
792
+
793
+ <span class='kw'>def</span> <span class='id identifier rubyid_after'>after</span>
794
+ <span class='kw'>end</span></pre>
795
+ </td>
796
+ </tr>
797
+ </table>
798
+ </div>
799
+
800
+ <div class="method_details ">
801
+ <h3 class="signature " id="before-instance_method">
802
+
803
+ - (<tt>Object</tt>) <strong>before</strong>
804
+
805
+
806
+
807
+
808
+
809
+ </h3><div class="docstring">
810
+ <div class="discussion">
811
+
812
+ <p>Filter run before Tango execution</p>
813
+
814
+
815
+ </div>
816
+ </div>
817
+ <div class="tags">
818
+
819
+
820
+ </div><table class="source_code">
821
+ <tr>
822
+ <td>
823
+ <pre class="lines">
824
+
825
+
826
+ 41
827
+ 42</pre>
828
+ </td>
829
+ <td>
830
+ <pre class="code"><span class="info file"># File 'lib/tango/app.rb', line 41</span>
831
+
832
+ <span class='kw'>def</span> <span class='id identifier rubyid_before'>before</span>
833
+ <span class='kw'>end</span></pre>
834
+ </td>
835
+ </tr>
836
+ </table>
837
+ </div>
838
+
839
+ <div class="method_details ">
840
+ <h3 class="signature " id="register_model-instance_method">
841
+
842
+ - (<tt>Object</tt>) <strong>register_model</strong>(symbol, model)
843
+
844
+
845
+
846
+
847
+
848
+ </h3><div class="docstring">
849
+ <div class="discussion">
850
+
851
+ <p>Register new resource model</p>
852
+
853
+
854
+ </div>
855
+ </div>
856
+ <div class="tags">
857
+ <p class="tag_title">Parameters:</p>
858
+ <ul class="param">
859
+
860
+ <li>
861
+
862
+ <span class='name'>symbol</span>
863
+
864
+
865
+ <span class='type'>(<tt>Symbol</tt>)</span>
866
+
867
+
868
+
869
+ </li>
870
+
871
+ <li>
872
+
873
+ <span class='name'>model</span>
874
+
875
+
876
+ <span class='type'>(<tt>Class</tt>)</span>
877
+
878
+
879
+
880
+ </li>
881
+
882
+ </ul>
883
+
884
+
885
+ </div><table class="source_code">
886
+ <tr>
887
+ <td>
888
+ <pre class="lines">
889
+
890
+
891
+ 52
892
+ 53
893
+ 54
894
+ 55
895
+ 56
896
+ 57
897
+ 58
898
+ 59
899
+ 60
900
+ 61</pre>
901
+ </td>
902
+ <td>
903
+ <pre class="code"><span class="info file"># File 'lib/tango/app.rb', line 52</span>
904
+
905
+ <span class='kw'>def</span> <span class='id identifier rubyid_register_model'>register_model</span><span class='lparen'>(</span> <span class='id identifier rubyid_symbol'>symbol</span><span class='comma'>,</span> <span class='id identifier rubyid_model'>model</span> <span class='rparen'>)</span>
906
+
907
+ <span class='ivar'>@models</span><span class='lbracket'>[</span><span class='id identifier rubyid_symbol'>symbol</span><span class='rbracket'>]</span> <span class='op'>=</span> <span class='id identifier rubyid_model'>model</span>
908
+
909
+ <span class='comment'># Truncate table of non persistent model
910
+ </span> <span class='kw'>unless</span> <span class='id identifier rubyid_model'>model</span><span class='period'>.</span><span class='id identifier rubyid_persistent?'>persistent?</span>
911
+ <span class='const'>ActiveRecord</span><span class='op'>::</span><span class='const'>Base</span><span class='period'>.</span><span class='id identifier rubyid_connection'>connection</span><span class='period'>.</span><span class='id identifier rubyid_execute'>execute</span><span class='lparen'>(</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>TRUNCATE </span><span class='embexpr_beg'>#{</span><span class='id identifier rubyid_model'>model</span><span class='period'>.</span><span class='id identifier rubyid_table_name'>table_name</span><span class='embexpr_end'>}</span><span class='tstring_end'>&quot;</span></span> <span class='rparen'>)</span>
912
+ <span class='kw'>end</span>
913
+
914
+ <span class='kw'>end</span></pre>
915
+ </td>
916
+ </tr>
917
+ </table>
918
+ </div>
919
+
920
+ <div class="method_details ">
921
+ <h3 class="signature " id="register_operator-instance_method">
922
+
923
+ - (<tt>Object</tt>) <strong>register_operator</strong>(symbol, operator)
924
+
925
+
926
+
927
+
928
+
929
+ </h3><div class="docstring">
930
+ <div class="discussion">
931
+
932
+ <p>Register new resource operator</p>
933
+
934
+
935
+ </div>
936
+ </div>
937
+ <div class="tags">
938
+ <p class="tag_title">Parameters:</p>
939
+ <ul class="param">
940
+
941
+ <li>
942
+
943
+ <span class='name'>symbol</span>
944
+
945
+
946
+ <span class='type'>(<tt>Symbol</tt>)</span>
947
+
948
+
949
+
950
+ </li>
951
+
952
+ <li>
953
+
954
+ <span class='name'>operator</span>
955
+
956
+
957
+ <span class='type'>(<tt>Class</tt>)</span>
958
+
959
+
960
+
961
+ </li>
962
+
963
+ </ul>
964
+
965
+
966
+ </div><table class="source_code">
967
+ <tr>
968
+ <td>
969
+ <pre class="lines">
970
+
971
+
972
+ 67
973
+ 68
974
+ 69
975
+ 70
976
+ 71
977
+ 72
978
+ 73
979
+ 74
980
+ 75
981
+ 76</pre>
982
+ </td>
983
+ <td>
984
+ <pre class="code"><span class="info file"># File 'lib/tango/app.rb', line 67</span>
985
+
986
+ <span class='kw'>def</span> <span class='id identifier rubyid_register_operator'>register_operator</span><span class='lparen'>(</span> <span class='id identifier rubyid_symbol'>symbol</span><span class='comma'>,</span> <span class='id identifier rubyid_operator'>operator</span> <span class='rparen'>)</span>
987
+
988
+ <span class='ivar'>@operators</span><span class='lbracket'>[</span><span class='id identifier rubyid_symbol'>symbol</span><span class='rbracket'>]</span> <span class='op'>=</span> <span class='id identifier rubyid_operator'>operator</span>
989
+
990
+ <span class='comment'># Register operator with resource cache system
991
+ </span> <span class='ivar'>@cache</span><span class='period'>.</span><span class='id identifier rubyid_register'>register</span><span class='lparen'>(</span> <span class='id identifier rubyid_symbol'>symbol</span> <span class='rparen'>)</span> <span class='kw'>do</span> <span class='op'>|</span><span class='id identifier rubyid_resource'>resource</span><span class='op'>|</span>
992
+ <span class='id identifier rubyid_operator'>operator</span><span class='period'>.</span><span class='id identifier rubyid_load'>load</span><span class='lparen'>(</span> <span class='id identifier rubyid_resource'>resource</span> <span class='rparen'>)</span>
993
+ <span class='kw'>end</span>
994
+
995
+ <span class='kw'>end</span></pre>
996
+ </td>
997
+ </tr>
998
+ </table>
999
+ </div>
1000
+
1001
+ <div class="method_details ">
1002
+ <h3 class="signature " id="run-instance_method">
1003
+
1004
+ - (<tt>Integer</tt>) <strong>run</strong>
1005
+
1006
+
1007
+
1008
+
1009
+
1010
+ </h3><div class="docstring">
1011
+ <div class="discussion">
1012
+
1013
+ <p>Run ETL process</p>
1014
+
1015
+
1016
+ </div>
1017
+ </div>
1018
+ <div class="tags">
1019
+ <p class="tag_title">Parameters:</p>
1020
+ <ul class="param">
1021
+
1022
+ <li>
1023
+
1024
+ <span class='name'>link_stack</span>
1025
+
1026
+
1027
+ <span class='type'>(<tt><span class='object_link'><a href="LinkStack.html" title="Tango::LinkStack (class)">Tango::LinkStack</a></span></tt>)</span>
1028
+
1029
+
1030
+
1031
+ </li>
1032
+
1033
+ <li>
1034
+
1035
+ <span class='name'>dispatcher</span>
1036
+
1037
+
1038
+ <span class='type'>(<tt>Tango::Etl::Dispatcher</tt>)</span>
1039
+
1040
+
1041
+
1042
+ </li>
1043
+
1044
+ <li>
1045
+
1046
+ <span class='name'>cache</span>
1047
+
1048
+
1049
+ <span class='type'>(<tt>Tango::Resources::Cache</tt>)</span>
1050
+
1051
+
1052
+
1053
+ </li>
1054
+
1055
+ <li>
1056
+
1057
+ <span class='name'>http_client</span>
1058
+
1059
+
1060
+ <span class='type'>(<tt>Object</tt>)</span>
1061
+
1062
+
1063
+
1064
+ &mdash;
1065
+ <div class='inline'>
1066
+ <p>Must implement get method</p>
1067
+ </div>
1068
+
1069
+ </li>
1070
+
1071
+ <li>
1072
+
1073
+ <span class='name'>parser</span>
1074
+
1075
+
1076
+ <span class='type'>(<tt>Object</tt>)</span>
1077
+
1078
+
1079
+
1080
+ &mdash;
1081
+ <div class='inline'>
1082
+ <p>Must implement parse method</p>
1083
+ </div>
1084
+
1085
+ </li>
1086
+
1087
+ <li>
1088
+
1089
+ <span class='name'>logger</span>
1090
+
1091
+
1092
+ <span class='type'>(<tt>Logger</tt>)</span>
1093
+
1094
+
1095
+
1096
+ </li>
1097
+
1098
+ </ul>
1099
+
1100
+ <p class="tag_title">Returns:</p>
1101
+ <ul class="return">
1102
+
1103
+ <li>
1104
+
1105
+
1106
+ <span class='type'>(<tt>Integer</tt>)</span>
1107
+
1108
+
1109
+
1110
+ </li>
1111
+
1112
+ </ul>
1113
+
1114
+ </div><table class="source_code">
1115
+ <tr>
1116
+ <td>
1117
+ <pre class="lines">
1118
+
1119
+
1120
+ 87
1121
+ 88
1122
+ 89
1123
+ 90
1124
+ 91
1125
+ 92
1126
+ 93
1127
+ 94
1128
+ 95
1129
+ 96
1130
+ 97
1131
+ 98
1132
+ 99
1133
+ 100
1134
+ 101
1135
+ 102
1136
+ 103
1137
+ 104
1138
+ 105
1139
+ 106
1140
+ 107
1141
+ 108
1142
+ 109
1143
+ 110
1144
+ 111
1145
+ 112
1146
+ 113
1147
+ 114
1148
+ 115
1149
+ 116
1150
+ 117
1151
+ 118
1152
+ 119
1153
+ 120
1154
+ 121
1155
+ 122
1156
+ 123
1157
+ 124
1158
+ 125
1159
+ 126
1160
+ 127
1161
+ 128
1162
+ 129
1163
+ 130
1164
+ 131
1165
+ 132
1166
+ 133
1167
+ 134
1168
+ 135
1169
+ 136
1170
+ 137
1171
+ 138
1172
+ 139
1173
+ 140
1174
+ 141
1175
+ 142
1176
+ 143
1177
+ 144
1178
+ 145
1179
+ 146
1180
+ 147
1181
+ 148
1182
+ 149
1183
+ 150
1184
+ 151
1185
+ 152
1186
+ 153
1187
+ 154
1188
+ 155
1189
+ 156
1190
+ 157
1191
+ 158
1192
+ 159
1193
+ 160
1194
+ 161
1195
+ 162
1196
+ 163
1197
+ 164
1198
+ 165
1199
+ 166
1200
+ 167
1201
+ 168
1202
+ 169
1203
+ 170
1204
+ 171
1205
+ 172
1206
+ 173
1207
+ 174
1208
+ 175
1209
+ 176
1210
+ 177
1211
+ 178
1212
+ 179
1213
+ 180
1214
+ 181
1215
+ 182
1216
+ 183</pre>
1217
+ </td>
1218
+ <td>
1219
+ <pre class="code"><span class="info file"># File 'lib/tango/app.rb', line 87</span>
1220
+
1221
+ <span class='kw'>def</span> <span class='id identifier rubyid_run'>run</span>
1222
+
1223
+ <span class='comment'># Save beginning time
1224
+ </span> <span class='id identifier rubyid_start_time'>start_time</span> <span class='op'>=</span> <span class='const'>Time</span><span class='period'>.</span><span class='id identifier rubyid_now'>now</span>
1225
+
1226
+ <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_info'>info</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Running Tango v.</span><span class='embexpr_beg'>#{</span><span class='const'>Tango</span><span class='op'>::</span><span class='const'>VERSION</span><span class='embexpr_end'>}</span><span class='tstring_content'> ...</span><span class='tstring_end'>&quot;</span></span>
1227
+ <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_info'>info</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Target: </span><span class='embexpr_beg'>#{</span><span class='ivar'>@link_stack</span><span class='period'>.</span><span class='id identifier rubyid_host'>host</span><span class='embexpr_end'>}</span><span class='tstring_content'>.</span><span class='tstring_end'>&quot;</span></span>
1228
+
1229
+ <span class='comment'># Use next unlocked database
1230
+ </span> <span class='const'>Multidb</span><span class='period'>.</span><span class='id identifier rubyid_use'>use</span><span class='lparen'>(</span> <span class='ivar'>@db_locker</span><span class='period'>.</span><span class='id identifier rubyid_unlocked'>unlocked</span> <span class='rparen'>)</span>
1231
+ <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_info'>info</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Using database &#39;</span><span class='embexpr_beg'>#{</span><span class='ivar'>@db_locker</span><span class='period'>.</span><span class='id identifier rubyid_unlocked'>unlocked</span><span class='embexpr_end'>}</span><span class='tstring_content'>&#39;.</span><span class='tstring_end'>&quot;</span></span>
1232
+
1233
+ <span class='comment'># Run before filter
1234
+ </span> <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_info'>info</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Loading cache ...</span><span class='tstring_end'>&quot;</span></span>
1235
+ <span class='id identifier rubyid_load_cache'>load_cache</span>
1236
+
1237
+ <span class='comment'># Run before filter
1238
+ </span> <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_info'>info</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Running before callback ...</span><span class='tstring_end'>&quot;</span></span>
1239
+ <span class='id identifier rubyid_before'>before</span>
1240
+
1241
+ <span class='comment'># Init counter of crawled links
1242
+ </span> <span class='id identifier rubyid_links_counter'>links_counter</span> <span class='op'>=</span> <span class='int'>0</span>
1243
+ <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_info'>info</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Tango starts crawling ...</span><span class='tstring_end'>&quot;</span></span>
1244
+
1245
+ <span class='comment'># Start crawling website
1246
+ </span> <span class='kw'>while</span><span class='lparen'>(</span> <span class='ivar'>@link_stack</span><span class='period'>.</span><span class='id identifier rubyid_has_links?'>has_links?</span> <span class='rparen'>)</span>
1247
+
1248
+ <span class='comment'># Get a link from the stack
1249
+ </span> <span class='id identifier rubyid_link'>link</span> <span class='op'>=</span> <span class='ivar'>@link_stack</span><span class='period'>.</span><span class='id identifier rubyid_shift'>shift</span>
1250
+
1251
+ <span class='comment'># Skip iteration if no handler found
1252
+ </span> <span class='kw'>if</span> <span class='op'>!</span> <span class='id identifier rubyid_handler_klass'>handler_klass</span> <span class='op'>=</span> <span class='ivar'>@dispatcher</span><span class='period'>.</span><span class='id identifier rubyid_find_handler'>find_handler</span><span class='lparen'>(</span> <span class='id identifier rubyid_link'>link</span> <span class='rparen'>)</span>
1253
+ <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_error'>error</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>No handler for link: </span><span class='embexpr_beg'>#{</span><span class='id identifier rubyid_link'>link</span><span class='embexpr_end'>}</span><span class='tstring_content'>.</span><span class='tstring_end'>&quot;</span></span>
1254
+ <span class='kw'>next</span>
1255
+ <span class='kw'>end</span>
1256
+
1257
+ <span class='comment'># Try to get contents of the link
1258
+ </span> <span class='kw'>begin</span>
1259
+ <span class='id identifier rubyid_response'>response</span> <span class='op'>=</span> <span class='ivar'>@http_client</span><span class='period'>.</span><span class='id identifier rubyid_get'>get</span><span class='lparen'>(</span> <span class='ivar'>@link_stack</span><span class='period'>.</span><span class='id identifier rubyid_host'>host</span> <span class='op'>+</span> <span class='id identifier rubyid_link'>link</span> <span class='rparen'>)</span>
1260
+ <span class='kw'>rescue</span> <span class='const'>StandardError</span> <span class='op'>=&gt;</span> <span class='id identifier rubyid_e'>e</span>
1261
+ <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_error'>error</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Could not download contents of </span><span class='embexpr_beg'>#{</span><span class='ivar'>@link_stack</span><span class='period'>.</span><span class='id identifier rubyid_host'>host</span> <span class='op'>+</span> <span class='id identifier rubyid_link'>link</span><span class='embexpr_end'>}</span><span class='tstring_content'> link.</span><span class='tstring_end'>&quot;</span></span><span class='semicolon'>;</span> <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_error'>error</span> <span class='id identifier rubyid_e'>e</span><span class='period'>.</span><span class='id identifier rubyid_message'>message</span>
1262
+ <span class='kw'>next</span>
1263
+ <span class='kw'>end</span>
1264
+
1265
+ <span class='comment'># Continue only when response has code 200 or 201
1266
+ </span> <span class='kw'>if</span> <span class='op'>!</span> <span class='lbracket'>[</span> <span class='int'>200</span><span class='comma'>,</span> <span class='int'>201</span> <span class='rbracket'>]</span><span class='period'>.</span><span class='id identifier rubyid_include?'>include?</span><span class='lparen'>(</span> <span class='id identifier rubyid_response'>response</span><span class='period'>.</span><span class='id identifier rubyid_code'>code</span> <span class='rparen'>)</span>
1267
+ <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_error'>error</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Response code for link </span><span class='embexpr_beg'>#{</span><span class='id identifier rubyid_link'>link</span><span class='embexpr_end'>}</span><span class='tstring_content'> is </span><span class='embexpr_beg'>#{</span><span class='id identifier rubyid_response'>response</span><span class='period'>.</span><span class='id identifier rubyid_code'>code</span><span class='embexpr_end'>}</span><span class='tstring_content'>. Only code 200 is accepted.</span><span class='tstring_end'>&quot;</span></span>
1268
+ <span class='kw'>next</span>
1269
+ <span class='kw'>end</span>
1270
+
1271
+ <span class='comment'># Parse response contents
1272
+ </span> <span class='id identifier rubyid_document'>document</span> <span class='op'>=</span> <span class='ivar'>@parser</span><span class='period'>.</span><span class='id identifier rubyid_parse'>parse</span><span class='lparen'>(</span> <span class='id identifier rubyid_response'>response</span><span class='period'>.</span><span class='id identifier rubyid_body'>body</span> <span class='rparen'>)</span>
1273
+ <span class='comment'># Init handler
1274
+ </span> <span class='id identifier rubyid_handler'>handler</span> <span class='op'>=</span> <span class='id identifier rubyid_handler_klass'>handler_klass</span><span class='period'>.</span><span class='id identifier rubyid_new'>new</span><span class='lparen'>(</span> <span class='id identifier rubyid_link'>link</span><span class='comma'>,</span> <span class='id identifier rubyid_document'>document</span><span class='comma'>,</span> <span class='ivar'>@cache</span> <span class='rparen'>)</span>
1275
+
1276
+ <span class='comment'># Append links fetched from handler
1277
+ </span> <span class='ivar'>@link_stack</span><span class='period'>.</span><span class='id identifier rubyid_append'>append</span><span class='lparen'>(</span> <span class='id identifier rubyid_handler'>handler</span><span class='period'>.</span><span class='id identifier rubyid_links'>links</span> <span class='rparen'>)</span>
1278
+
1279
+ <span class='comment'># Try to fire the handler
1280
+ </span> <span class='kw'>begin</span>
1281
+ <span class='id identifier rubyid_handler'>handler</span><span class='period'>.</span><span class='id identifier rubyid_trigger'>trigger</span>
1282
+ <span class='kw'>rescue</span> <span class='const'>StandardError</span> <span class='op'>=&gt;</span> <span class='id identifier rubyid_e'>e</span>
1283
+ <span class='comment'># Log error
1284
+ </span> <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_error'>error</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Link: </span><span class='embexpr_beg'>#{</span><span class='id identifier rubyid_link'>link</span><span class='embexpr_end'>}</span><span class='tstring_content'>. Handler had some troubles.</span><span class='tstring_end'>&quot;</span></span>
1285
+ <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_error'>error</span> <span class='id identifier rubyid_e'>e</span><span class='period'>.</span><span class='id identifier rubyid_message'>message</span>
1286
+ <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_error'>error</span> <span class='id identifier rubyid_e'>e</span><span class='period'>.</span><span class='id identifier rubyid_backtrace'>backtrace</span><span class='period'>.</span><span class='id identifier rubyid_join'>join</span><span class='lparen'>(</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>\n</span><span class='tstring_end'>&quot;</span></span> <span class='rparen'>)</span>
1287
+ <span class='kw'>else</span>
1288
+ <span class='id identifier rubyid_links_counter'>links_counter</span> <span class='op'>+=</span> <span class='int'>1</span>
1289
+ <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_debug'>debug</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Link: </span><span class='embexpr_beg'>#{</span><span class='id identifier rubyid_link'>link</span><span class='embexpr_end'>}</span><span class='tstring_content'>. Handler triggered successfully.</span><span class='tstring_end'>&quot;</span></span>
1290
+ <span class='kw'>end</span>
1291
+
1292
+ <span class='comment'># Sleep to give crawled server time to breath
1293
+ </span> <span class='id identifier rubyid_sleep'>sleep</span><span class='lparen'>(</span> <span class='ivar'>@config</span><span class='lbracket'>[</span><span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>sleep</span><span class='tstring_end'>&quot;</span></span><span class='rbracket'>]</span> <span class='op'>||</span> <span class='int'>0</span> <span class='rparen'>)</span>
1294
+
1295
+ <span class='kw'>end</span>
1296
+
1297
+ <span class='comment'># Release buffers
1298
+ </span> <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_info'>info</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Releasing buffers ...</span><span class='tstring_end'>&quot;</span></span>
1299
+ <span class='ivar'>@cache</span><span class='period'>.</span><span class='id identifier rubyid_buffer'>buffer</span><span class='period'>.</span><span class='id identifier rubyid_release_all'>release_all</span><span class='lparen'>(</span><span class='rparen'>)</span>
1300
+
1301
+ <span class='comment'># Run after filter
1302
+ </span> <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_info'>info</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Running after callback ...</span><span class='tstring_end'>&quot;</span></span>
1303
+ <span class='id identifier rubyid_after'>after</span>
1304
+
1305
+ <span class='comment'># Lock database used in this Tango iteration
1306
+ </span> <span class='ivar'>@db_locker</span><span class='period'>.</span><span class='id identifier rubyid_lock'>lock</span><span class='lparen'>(</span> <span class='ivar'>@db_locker</span><span class='period'>.</span><span class='id identifier rubyid_unlocked'>unlocked</span> <span class='rparen'>)</span>
1307
+
1308
+ <span class='comment'># Get time of script execution ending
1309
+ </span> <span class='id identifier rubyid_end_time'>end_time</span> <span class='op'>=</span> <span class='const'>Time</span><span class='period'>.</span><span class='id identifier rubyid_now'>now</span>
1310
+
1311
+ <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_info'>info</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Tango crawled </span><span class='embexpr_beg'>#{</span><span class='id identifier rubyid_links_counter'>links_counter</span><span class='embexpr_end'>}</span><span class='tstring_content'>/</span><span class='embexpr_beg'>#{</span><span class='ivar'>@link_stack</span><span class='period'>.</span><span class='id identifier rubyid_shifted'>shifted</span><span class='embexpr_end'>}</span><span class='tstring_content'> links successfully.</span><span class='tstring_end'>&quot;</span></span>
1312
+ <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_info'>info</span> <span class='tstring'><span class='tstring_beg'>&quot;</span><span class='tstring_content'>Start time: </span><span class='embexpr_beg'>#{</span><span class='id identifier rubyid_start_time'>start_time</span><span class='embexpr_end'>}</span><span class='tstring_content'>, end time: </span><span class='embexpr_beg'>#{</span><span class='id identifier rubyid_end_time'>end_time</span><span class='embexpr_end'>}</span><span class='tstring_content'>, time elapsed: </span><span class='embexpr_beg'>#{</span><span class='id identifier rubyid_end_time'>end_time</span> <span class='op'>-</span> <span class='id identifier rubyid_start_time'>start_time</span><span class='embexpr_end'>}</span><span class='tstring_content'> seconds.</span><span class='tstring_end'>&quot;</span></span>
1313
+
1314
+ <span class='comment'># Close logger
1315
+ </span> <span class='ivar'>@logger</span><span class='period'>.</span><span class='id identifier rubyid_close'>close</span>
1316
+
1317
+ <span class='kw'>end</span></pre>
1318
+ </td>
1319
+ </tr>
1320
+ </table>
1321
+ </div>
1322
+
1323
+ </div>
1324
+
1325
+ </div>
1326
+
1327
+ <div id="footer">
1328
+ Generated on Tue Apr 15 02:47:42 2014 by
1329
+ <a href="http://yardoc.org" title="Yay! A Ruby Documentation Tool" target="_parent">yard</a>
1330
+ 0.8.7.4 (ruby-2.0.0).
1331
+ </div>
1332
+
1333
+ </body>
1334
+ </html>