ghtorrent 0.4 → 0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. data/CHANGELOG +24 -0
  2. data/Gemfile +17 -0
  3. data/Gemfile.lock +40 -0
  4. data/README.md +23 -22
  5. data/bin/ght-data-retrieval +66 -24
  6. data/bin/ght-load +41 -19
  7. data/bin/ght-mirror-events +13 -16
  8. data/bin/ght-rm-dupl +119 -77
  9. data/lib/ghtorrent.rb +14 -4
  10. data/lib/ghtorrent/adapters/base_adapter.rb +17 -5
  11. data/lib/ghtorrent/adapters/mongo_persister.rb +122 -56
  12. data/lib/ghtorrent/api_client.rb +151 -16
  13. data/lib/ghtorrent/bson_orderedhash.rb +23 -0
  14. data/lib/ghtorrent/cache.rb +97 -0
  15. data/lib/ghtorrent/command.rb +43 -25
  16. data/lib/ghtorrent/gh_torrent_exception.rb +6 -0
  17. data/lib/ghtorrent/ghtorrent.rb +615 -164
  18. data/lib/ghtorrent/hash.rb +11 -0
  19. data/lib/ghtorrent/logging.rb +11 -7
  20. data/lib/ghtorrent/migrations/001_init_schema.rb +3 -3
  21. data/lib/ghtorrent/migrations/002_add_external_ref_ids.rb +2 -0
  22. data/lib/ghtorrent/migrations/003_add_orgs.rb +4 -1
  23. data/lib/ghtorrent/migrations/004_add_commit_comments.rb +4 -2
  24. data/lib/ghtorrent/migrations/005_add_repo_collaborators.rb +2 -0
  25. data/lib/ghtorrent/migrations/006_add_watchers.rb +2 -0
  26. data/lib/ghtorrent/migrations/007_add_pull_requests.rb +64 -0
  27. data/lib/ghtorrent/migrations/008_add_project_unq.rb +23 -0
  28. data/lib/ghtorrent/migrations/009_add_project_commit.rb +27 -0
  29. data/lib/ghtorrent/migrations/010_add_forks.rb +28 -0
  30. data/lib/ghtorrent/migrations/mysql_defaults.rb +6 -0
  31. data/lib/ghtorrent/persister.rb +3 -0
  32. data/lib/ghtorrent/retriever.rb +298 -102
  33. data/lib/ghtorrent/settings.rb +20 -1
  34. data/lib/ghtorrent/time.rb +5 -0
  35. data/lib/ghtorrent/utils.rb +22 -4
  36. data/lib/version.rb +5 -0
  37. metadata +173 -145
  38. data/lib/ghtorrent/call_stack.rb +0 -91
data/lib/ghtorrent/call_stack.rb DELETED
@@ -1,91 +0,0 @@
1
- module GHTorrent
2
- class CallStack
3
-
4
- @@callstacks = Hash.new
5
-
6
- attr_reader :name
7
-
8
- def self.new(*args)
9
- name = args[0]
10
- if @@callstacks.has_key? name
11
- @@callstacks[name]
12
- else
13
- o = allocate
14
- if o.__send__(:initialize, *args)
15
- @@callstacks[name] = o
16
- o
17
- else
18
- nil
19
- end
20
- end
21
- end
22
-
23
- def initialize(name, sync_every = 5)
24
-
25
- @stack = Array.new
26
- @name = name
27
- @sync = sync_every
28
-
29
- if File.exists?(name)
30
- @file = File.new(name, "r")
31
- puts "File #{name} exists, importing stack..."
32
- read = @file.readlines.reverse.reduce(0) { |acc, x|
33
- @stack.push x
34
- acc
35
- }
36
- puts "\n#{read} entries read"
37
- @file.close
38
- end
39
-
40
- flusher = Thread.new {
41
- while true
42
- begin
43
- if not @stack.empty?
44
- @file = File.new(name, "w+")
45
- @stack.each { |l| @file.write("#{l} \n") }
46
- @file.fsync
47
- @file.close
48
- end
49
- sleep(@sync)
50
- rescue
51
- puts "flusher thread failed for #{name}"
52
- end
53
- end
54
- }
55
-
56
- ObjectSpace.define_finalizer(self, proc {
57
- puts "Finalizer: Cleaning up #{@name}"
58
- @@callstacks.delete[@name]
59
- flusher.stop
60
- cleanup
61
- })
62
-
63
- at_exit { cleanup }
64
- end
65
-
66
- def push(item)
67
- @stack.push(item)
68
- end
69
-
70
- def pop()
71
- @stack.pop
72
- end
73
-
74
- def empty
75
- @stack.delete_if { |x| true }
76
- end
77
-
78
- private
79
-
80
- def cleanup
81
- if @stack.empty?
82
- if File.exists? @name
83
- puts "removing stack #{@name}"
84
- File.delete(@name)
85
- end
86
- else
87
- puts "stack #{@name} contains #{@stack.size} items"
88
- end
89
- end
90
- end
91
- end