zlib-socs 0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/Rakefile +0 -0
- data/bin/fetch-vk-accs +37 -0
- data/lib/fetch-ships.rb +52 -0
- data/lib/watch-ships.rb +81 -0
- data/tests/fetch-ships.rb +37 -0
- data/tests/watch-ships.rb +81 -0
- metadata +57 -0
data/Rakefile
ADDED
File without changes
|
data/bin/fetch-vk-accs
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
#encoding: utf-8
|
3
|
+
require 'bundler/setup'
|
4
|
+
require 'vk-watchdog'
|
5
|
+
##
|
6
|
+
# рекурсивно вытаскиваем инфу и друзей пользователей ВК на заданную глубину
|
7
|
+
#
|
8
|
+
|
9
|
+
if ARGV.empty?
|
10
|
+
dlog 'Command: %s depth start_uids
|
11
|
+
Example: fetch-ships.rb 2 1 #=> friends of user 1 and friends of friends of user 1'%[__FILE__]
|
12
|
+
exit 1
|
13
|
+
end
|
14
|
+
|
15
|
+
depth = ARGV.shift.to_i
|
16
|
+
uids = ARGV.map{|a| a.to_i}
|
17
|
+
|
18
|
+
dlog 'Fetching vk users infos and friends for depth %s from start uids: %s'%[depth, uids]
|
19
|
+
next_uids = uids.dup
|
20
|
+
uinfos = {}
|
21
|
+
ufriends = {}
|
22
|
+
0.upto(depth) do |loop_i|
|
23
|
+
dlog 'Have %s next uids, left %s loops'%[next_uids.size, depth-loop_i]
|
24
|
+
ufriends_uids = next_uids.select{|uid| not ufriends[uid]}
|
25
|
+
ufriends.merge! ZVK::users_friends(ufriends_uids)
|
26
|
+
uinfos_uids = (ufriends.keys+ufriends.values.flatten.uniq)
|
27
|
+
#.flatten.flatten.select{|uid| not uinfos[uid]}
|
28
|
+
uinfos.merge! ZVK::users_info(uinfos_uids)
|
29
|
+
next_uids = ufriends.to_a.flatten.flatten.uniq.select{|uid| not ufriends[uid] and uinfos[uid]}
|
30
|
+
|
31
|
+
end
|
32
|
+
|
33
|
+
|
34
|
+
puts uinfos.to_json
|
35
|
+
puts ufriends.to_json
|
36
|
+
|
37
|
+
dlog 'Total fetched: infos for %s users, friends for %s'%[uinfos.size, ufriends.size]
|
data/lib/fetch-ships.rb
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
#encoding: utf-8
|
3
|
+
require 'bundler/setup'
|
4
|
+
require 'vk-ruby'
|
5
|
+
|
6
|
+
|
7
|
+
module ZVK
|
8
|
+
##
|
9
|
+
# рекурсивно вытаскиваем инфу и друзей пользователей ВК на заданную глубину
|
10
|
+
#
|
11
|
+
|
12
|
+
def self.dlog(*args)
|
13
|
+
STDERR.puts args
|
14
|
+
end
|
15
|
+
|
16
|
+
|
17
|
+
##
|
18
|
+
# получить инфу по пользователям uids
|
19
|
+
# = returns
|
20
|
+
# хэш с инфой
|
21
|
+
def self.users_info(uids)
|
22
|
+
dlog 'infos in: %s'%[uids]
|
23
|
+
vk=VK::Application.new
|
24
|
+
res={}
|
25
|
+
[uids].flatten.each_slice(10000) do |next_uids|
|
26
|
+
begin
|
27
|
+
uinfos=vk.users.get('user_ids'=>next_uids.join(','),
|
28
|
+
fields: 'sex, bdate, city, country, photo_50, photo_100, photo_200_orig, photo_200, photo_400_orig, photo_max, photo_max_orig, online, online_mobile, lists, domain, has_mobile, contacts, connections, site, education, universities, schools, can_post, can_see_all_posts, can_see_audio, can_write_private_message, status, last_seen, relation, relatives, counters '.split(', '))
|
29
|
+
uinfos.each do |uinfo|
|
30
|
+
res[uinfo['uid']]=uinfo.dup if uinfo['uid']
|
31
|
+
end
|
32
|
+
rescue => err
|
33
|
+
end
|
34
|
+
end
|
35
|
+
res
|
36
|
+
end
|
37
|
+
|
38
|
+
# для каждого айдишника подтягиваем друзей
|
39
|
+
# результат - хэш: айди->друзья(их айдишники)
|
40
|
+
def self.users_friends(uids)
|
41
|
+
dlog 'friends in: %s'%[uids]
|
42
|
+
vk=VK::Application.new
|
43
|
+
Hash[
|
44
|
+
uids.map do |uid|
|
45
|
+
begin
|
46
|
+
[uid, vk.friends.get(uid: uid)]
|
47
|
+
rescue =>err
|
48
|
+
end
|
49
|
+
end
|
50
|
+
]
|
51
|
+
end
|
52
|
+
end
|
data/lib/watch-ships.rb
ADDED
@@ -0,0 +1,81 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
#encoding: utf-8
|
3
|
+
require 'bundler/setup'
|
4
|
+
require 'vk-ruby'
|
5
|
+
require 'mongo'
|
6
|
+
require 'set'
|
7
|
+
require 'json'
|
8
|
+
|
9
|
+
##
|
10
|
+
# мониторим наших овечек и записываем все их изменения/обновления:
|
11
|
+
# 1. Подтягиваем из монги список друзей, профилей для мониторинга
|
12
|
+
# 2. Для всех получаем актуальные данные, сравниваем - если ==, то пропускаем
|
13
|
+
# если расходятся - тогда обновляем запись в монге и пишем в лог инфу об изменении
|
14
|
+
|
15
|
+
def dlog(*args)
|
16
|
+
STDERR.puts args
|
17
|
+
end
|
18
|
+
|
19
|
+
def info_diff(uid, info, ninfo)
|
20
|
+
diffs=[]
|
21
|
+
Set.new(infos.keys+ninfos.keys).each do |fld|
|
22
|
+
ofval, nfval = infos[fld], ninfos[fld]
|
23
|
+
next if ofval == nfval
|
24
|
+
if not ofval # значит до обновления филда не было
|
25
|
+
diffs << {at: Time.now.to_i, diff: 'new field', field: {fld=>nfval}}
|
26
|
+
#ninfos_col.update({uid: uid}, {'$set'=>diffs.last}) # и в бд
|
27
|
+
elsif not nfval # а теперь филд удалился
|
28
|
+
diffs << {at: Time.now.to_i, diff: 'del field', field: {fld=>ofval}}
|
29
|
+
#! в базе не оставляем
|
30
|
+
#ninfos_col.update({uid: uid}, {'$set'=>diffs.last}) # и в бд
|
31
|
+
elsif nfval != ofval
|
32
|
+
diffs << {at: Time.now.to_i, diff: 'upd field', field: fld, vals: [ofval,nfval]} # филд+старое и новое
|
33
|
+
end
|
34
|
+
end
|
35
|
+
diffs
|
36
|
+
end
|
37
|
+
|
38
|
+
def friends_diff(uid, friends, nfriends)
|
39
|
+
f, nf = Set.new(friends), Set.new(nfriends)
|
40
|
+
return [] if (f==nf)
|
41
|
+
|
42
|
+
diffs=[]
|
43
|
+
# new friends
|
44
|
+
diffs << (nf-f).map{|fuid| {at: Time.now.to_i, diff: 'new friends', friends: [uid, fuid]} }
|
45
|
+
diffs << (f-nf).map{|fuid| {at: Time.now.to_i, diff: 'del friends', friends: [uid, fuid]} }
|
46
|
+
diffs
|
47
|
+
end
|
48
|
+
|
49
|
+
|
50
|
+
mng=Mongo::MongoClient.new
|
51
|
+
wdb = mng['vk_wdog']
|
52
|
+
uinfos_col = wdb['users_info']
|
53
|
+
friends_col = wdb['users_info']
|
54
|
+
# делаем дело
|
55
|
+
# берём всех засветившихся uid
|
56
|
+
uids = Set.new(uinfos_col.distinct('uid')+
|
57
|
+
friends_col.distinct('uid')+
|
58
|
+
friends_col.distinct('friends'))
|
59
|
+
|
60
|
+
fetch_ships_cmd = './fetch-ships.rb 2 3122378 | grep -v DEBUG > watchdog.updates'
|
61
|
+
uinfos = system( fetch_ships_cmd )# '%s 1 %s |grep -v DEBUG > watchdog.updates'%[fetch_ships_cmd, uids.to_a.join(' ')])
|
62
|
+
|
63
|
+
ninfos, nfriends = IO.read('watchdog.updates').split("\n").map{|l| JSON::parse(l)}
|
64
|
+
ninfos_col, friends_col = wdb[(uinfos_col.name+'_last')], wdb[(friends_col.name+'_last')]
|
65
|
+
ninfos_col << ninfos
|
66
|
+
nfriends_col << nfriends
|
67
|
+
nuids = Set.new(ninfos_col.distinct('uid')+
|
68
|
+
nfriends_col.distinct('uid')+
|
69
|
+
nfriends_col.distinct('friends'))
|
70
|
+
|
71
|
+
nuids.each do |uid|
|
72
|
+
# вычисляем изменения в данных
|
73
|
+
idiffs, fdiffs= info_diff(uid, uinfos[uid], ninfos[uid]), friends_diff(uid, friends[uid], nfriends[uid] )
|
74
|
+
wdb[uinfos_col.name+'_diffs'] << idiffs
|
75
|
+
wdb[friends_col.name+'_diffs'] << fdiffs
|
76
|
+
end
|
77
|
+
wdb.drop(uinfos_col.name)
|
78
|
+
wdb.drop(friends_col.name)
|
79
|
+
wdb.rename(ninfos_col.name, uinfos_col.name)
|
80
|
+
wdb.rename(nfriends_col.name, friends_col.name)
|
81
|
+
dlog 'Total fetched: infos for %s users, friends for %s'%[ninfos.size, nfriends.size]
|
@@ -0,0 +1,37 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
#encoding: utf-8
|
3
|
+
require 'bundler/setup'
|
4
|
+
require 'vk-watchdog'
|
5
|
+
##
|
6
|
+
# рекурсивно вытаскиваем инфу и друзей пользователей ВК на заданную глубину
|
7
|
+
#
|
8
|
+
|
9
|
+
if ARGV.empty?
|
10
|
+
dlog 'Command: %s depth start_uids
|
11
|
+
Example: fetch-ships.rb 2 1 #=> friends of user 1 and friends of friends of user 1'%[__FILE__]
|
12
|
+
exit 1
|
13
|
+
end
|
14
|
+
|
15
|
+
depth = ARGV.shift.to_i
|
16
|
+
uids = ARGV.map{|a| a.to_i}
|
17
|
+
|
18
|
+
dlog 'Fetching vk users infos and friends for depth %s from start uids: %s'%[depth, uids]
|
19
|
+
next_uids = uids.dup
|
20
|
+
uinfos = {}
|
21
|
+
ufriends = {}
|
22
|
+
0.upto(depth) do |loop_i|
|
23
|
+
dlog 'Have %s next uids, left %s loops'%[next_uids.size, depth-loop_i]
|
24
|
+
ufriends_uids = next_uids.select{|uid| not ufriends[uid]}
|
25
|
+
ufriends.merge! ZVK::users_friends(ufriends_uids)
|
26
|
+
uinfos_uids = (ufriends.keys+ufriends.values.flatten.uniq)
|
27
|
+
#.flatten.flatten.select{|uid| not uinfos[uid]}
|
28
|
+
uinfos.merge! ZVK::users_info(uinfos_uids)
|
29
|
+
next_uids = ufriends.to_a.flatten.flatten.uniq.select{|uid| not ufriends[uid] and uinfos[uid]}
|
30
|
+
|
31
|
+
end
|
32
|
+
|
33
|
+
|
34
|
+
puts uinfos.to_json
|
35
|
+
puts ufriends.to_json
|
36
|
+
|
37
|
+
dlog 'Total fetched: infos for %s users, friends for %s'%[uinfos.size, ufriends.size]
|
@@ -0,0 +1,81 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
#encoding: utf-8
|
3
|
+
require 'bundler/setup'
|
4
|
+
require 'vk-ruby'
|
5
|
+
require 'mongo'
|
6
|
+
require 'set'
|
7
|
+
require 'json'
|
8
|
+
|
9
|
+
##
|
10
|
+
# мониторим наших овечек и записываем все их изменения/обновления:
|
11
|
+
# 1. Подтягиваем из монги список друзей, профилей для мониторинга
|
12
|
+
# 2. Для всех получаем актуальные данные, сравниваем - если ==, то пропускаем
|
13
|
+
# если расходятся - тогда обновляем запись в монге и пишем в лог инфу об изменении
|
14
|
+
|
15
|
+
def dlog(*args)
|
16
|
+
STDERR.puts args
|
17
|
+
end
|
18
|
+
|
19
|
+
def info_diff(uid, info, ninfo)
|
20
|
+
diffs=[]
|
21
|
+
Set.new(infos.keys+ninfos.keys).each do |fld|
|
22
|
+
ofval, nfval = infos[fld], ninfos[fld]
|
23
|
+
next if ofval == nfval
|
24
|
+
if not ofval # значит до обновления филда не было
|
25
|
+
diffs << {at: Time.now.to_i, diff: 'new field', field: {fld=>nfval}}
|
26
|
+
#ninfos_col.update({uid: uid}, {'$set'=>diffs.last}) # и в бд
|
27
|
+
elsif not nfval # а теперь филд удалился
|
28
|
+
diffs << {at: Time.now.to_i, diff: 'del field', field: {fld=>ofval}}
|
29
|
+
#! в базе не оставляем
|
30
|
+
#ninfos_col.update({uid: uid}, {'$set'=>diffs.last}) # и в бд
|
31
|
+
elsif nfval != ofval
|
32
|
+
diffs << {at: Time.now.to_i, diff: 'upd field', field: fld, vals: [ofval,nfval]} # филд+старое и новое
|
33
|
+
end
|
34
|
+
end
|
35
|
+
diffs
|
36
|
+
end
|
37
|
+
|
38
|
+
def friends_diff(uid, friends, nfriends)
|
39
|
+
f, nf = Set.new(friends), Set.new(nfriends)
|
40
|
+
return [] if (f==nf)
|
41
|
+
|
42
|
+
diffs=[]
|
43
|
+
# new friends
|
44
|
+
diffs << (nf-f).map{|fuid| {at: Time.now.to_i, diff: 'new friends', friends: [uid, fuid]} }
|
45
|
+
diffs << (f-nf).map{|fuid| {at: Time.now.to_i, diff: 'del friends', friends: [uid, fuid]} }
|
46
|
+
diffs
|
47
|
+
end
|
48
|
+
|
49
|
+
|
50
|
+
mng=Mongo::MongoClient.new
|
51
|
+
wdb = mng['vk_wdog']
|
52
|
+
uinfos_col = wdb['users_info']
|
53
|
+
friends_col = wdb['users_info']
|
54
|
+
# делаем дело
|
55
|
+
# берём всех засветившихся uid
|
56
|
+
uids = Set.new(uinfos_col.distinct('uid')+
|
57
|
+
friends_col.distinct('uid')+
|
58
|
+
friends_col.distinct('friends'))
|
59
|
+
|
60
|
+
fetch_ships_cmd = './fetch-ships.rb 2 3122378 | grep -v DEBUG > watchdog.updates'
|
61
|
+
uinfos = system( fetch_ships_cmd )# '%s 1 %s |grep -v DEBUG > watchdog.updates'%[fetch_ships_cmd, uids.to_a.join(' ')])
|
62
|
+
|
63
|
+
ninfos, nfriends = IO.read('watchdog.updates').split("\n").map{|l| JSON::parse(l)}
|
64
|
+
ninfos_col, friends_col = wdb[(uinfos_col.name+'_last')], wdb[(friends_col.name+'_last')]
|
65
|
+
ninfos_col << ninfos
|
66
|
+
nfriends_col << nfriends
|
67
|
+
nuids = Set.new(ninfos_col.distinct('uid')+
|
68
|
+
nfriends_col.distinct('uid')+
|
69
|
+
nfriends_col.distinct('friends'))
|
70
|
+
|
71
|
+
nuids.each do |uid|
|
72
|
+
# вычисляем изменения в данных
|
73
|
+
idiffs, fdiffs= info_diff(uid, uinfos[uid], ninfos[uid]), friends_diff(uid, friends[uid], nfriends[uid] )
|
74
|
+
wdb[uinfos_col.name+'_diffs'] << idiffs
|
75
|
+
wdb[friends_col.name+'_diffs'] << fdiffs
|
76
|
+
end
|
77
|
+
wdb.drop(uinfos_col.name)
|
78
|
+
wdb.drop(friends_col.name)
|
79
|
+
wdb.rename(ninfos_col.name, uinfos_col.name)
|
80
|
+
wdb.rename(nfriends_col.name, friends_col.name)
|
81
|
+
dlog 'Total fetched: infos for %s users, friends for %s'%[ninfos.size, nfriends.size]
|
metadata
ADDED
@@ -0,0 +1,57 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: zlib-socs
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: '0.1'
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- ev42
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2013-12-23 00:00:00.000000000 Z
|
13
|
+
dependencies: []
|
14
|
+
description: ! 'ZLibs::Социализм
|
15
|
+
|
16
|
+
Простая либа для работы с ресурсами социализма.
|
17
|
+
|
18
|
+
Разбавляется методами по мере потребновсти в них.
|
19
|
+
|
20
|
+
'
|
21
|
+
email: splin.ev42@gmail.com
|
22
|
+
executables: []
|
23
|
+
extensions: []
|
24
|
+
extra_rdoc_files: []
|
25
|
+
files:
|
26
|
+
- lib/fetch-ships.rb
|
27
|
+
- lib/watch-ships.rb
|
28
|
+
- bin/fetch-vk-accs
|
29
|
+
- tests/fetch-ships.rb
|
30
|
+
- tests/watch-ships.rb
|
31
|
+
- Rakefile
|
32
|
+
homepage: http://google.com
|
33
|
+
licenses:
|
34
|
+
- none
|
35
|
+
post_install_message:
|
36
|
+
rdoc_options: []
|
37
|
+
require_paths:
|
38
|
+
- lib
|
39
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
40
|
+
none: false
|
41
|
+
requirements:
|
42
|
+
- - ! '>='
|
43
|
+
- !ruby/object:Gem::Version
|
44
|
+
version: '0'
|
45
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
46
|
+
none: false
|
47
|
+
requirements:
|
48
|
+
- - ! '>='
|
49
|
+
- !ruby/object:Gem::Version
|
50
|
+
version: '0'
|
51
|
+
requirements: []
|
52
|
+
rubyforge_project:
|
53
|
+
rubygems_version: 1.8.23
|
54
|
+
signing_key:
|
55
|
+
specification_version: 3
|
56
|
+
summary: zlibs::vk
|
57
|
+
test_files: []
|