vk-watchdog 0.1

data/bin/fetch-vk-accs

@@ -0,0 +1,37 @@
+#!/usr/bin/ruby
+#encoding: utf-8
+require 'bundler/setup'
+require 'vk-watchdog'
+##
+# рекурсивно вытаскиваем инфу и друзей пользователей ВК на заданную глубину
+# 
+
+if ARGV.empty?
+	dlog 'Command: %s depth start_uids 
+Example: fetch-ships.rb 2 1  #=> friends of user 1 and friends of friends of user 1'%[__FILE__]
+	exit 1
+end
+
+depth = ARGV.shift.to_i
+uids = ARGV.map{|a| a.to_i}
+
+dlog 'Fetching vk users infos and friends for depth %s from start uids: %s'%[depth, uids]
+next_uids = uids.dup
+uinfos = {}
+ufriends = {}
+0.upto(depth) do |loop_i|
+	dlog 'Have %s next uids, left %s loops'%[next_uids.size, depth-loop_i]
+	ufriends_uids = next_uids.select{|uid| not ufriends[uid]}
+	ufriends.merge! ZVK::users_friends(ufriends_uids)
+	uinfos_uids = (ufriends.keys+ufriends.values.flatten.uniq)
+#.flatten.flatten.select{|uid| not uinfos[uid]}
+	uinfos.merge! ZVK::users_info(uinfos_uids)
+    	next_uids = ufriends.to_a.flatten.flatten.uniq.select{|uid| not ufriends[uid] and uinfos[uid]}
+
+end
+
+
+puts uinfos.to_json
+puts ufriends.to_json
+
+dlog 'Total fetched: infos for %s users, friends for %s'%[uinfos.size, ufriends.size]

data/lib/fetch-ships.rb

@@ -0,0 +1,52 @@
+#!/usr/bin/ruby
+#encoding: utf-8
+require 'bundler/setup'
+require 'vk-ruby'
+
+
+module ZVK
+##
+# рекурсивно вытаскиваем инфу и друзей пользователей ВК на заданную глубину
+# 
+
+    def self.dlog(*args)
+	STDERR.puts args
+    end
+
+
+## 
+# получить инфу по пользователям uids
+# = returns
+# хэш с инфой
+    def self.users_info(uids)
+	    dlog 'infos in: %s'%[uids]
+	    vk=VK::Application.new
+	    res={}
+	    [uids].flatten.each_slice(10000) do |next_uids|
+		    begin
+			uinfos=vk.users.get('user_ids'=>next_uids.join(','),
+				      fields: 'sex, bdate, city, country, photo_50, photo_100, photo_200_orig, photo_200, photo_400_orig, photo_max, photo_max_orig, online, online_mobile, lists, domain, has_mobile, contacts, connections, site, education, universities, schools, can_post, can_see_all_posts, can_see_audio, can_write_private_message, status, last_seen, relation, relatives, counters '.split(', '))
+			uinfos.each do |uinfo|
+				res[uinfo['uid']]=uinfo.dup if uinfo['uid']
+			end
+		    rescue => err
+		    end
+	    end
+	    res	     
+    end
+
+# для каждого айдишника подтягиваем друзей
+# результат - хэш: айди->друзья(их айдишники)
+    def self.users_friends(uids)
+	    dlog 'friends in: %s'%[uids]
+	    vk=VK::Application.new
+	    Hash[
+		uids.map do |uid|
+		  begin
+		    [uid, vk.friends.get(uid: uid)]
+		  rescue =>err
+		  end
+		end 
+	    ]
+    end
+end

data/lib/watch-ships.rb

@@ -0,0 +1,81 @@
+#!/usr/bin/ruby
+#encoding: utf-8
+require 'bundler/setup'
+require 'vk-ruby'
+require 'mongo'
+require 'set'
+require 'json'
+
+##
+# мониторим наших овечек и записываем все их изменения/обновления:
+# 1. Подтягиваем из монги список друзей, профилей для мониторинга
+# 2.   Для всех получаем актуальные данные, сравниваем - если ==, то пропускаем
+#    если расходятся - тогда обновляем запись в монге и пишем в лог инфу об изменении
+
+def dlog(*args)
+    STDERR.puts args
+end
+
+def info_diff(uid, info, ninfo)
+    diffs=[]
+    Set.new(infos.keys+ninfos.keys).each do |fld|
+    	ofval, nfval = infos[fld], ninfos[fld]
+	next if ofval == nfval
+	if not ofval # значит до обновления филда не было
+		diffs << {at: Time.now.to_i, diff: 'new field', field: {fld=>nfval}}	
+#ninfos_col.update({uid: uid}, {'$set'=>diffs.last}) # и в бд
+	elsif not nfval # а теперь филд удалился
+		diffs << {at: Time.now.to_i, diff: 'del field', field: {fld=>ofval}}
+                #! в базе не оставляем
+#ninfos_col.update({uid: uid}, {'$set'=>diffs.last}) # и в бд
+	elsif nfval != ofval
+		diffs << {at: Time.now.to_i, diff: 'upd field', field: fld, vals: [ofval,nfval]} # филд+старое и новое
+	end
+    end
+    diffs
+end
+
+def friends_diff(uid, friends, nfriends)
+    f, nf = Set.new(friends), Set.new(nfriends)
+    return [] if (f==nf)
+
+    diffs=[]
+# new friends
+    diffs << (nf-f).map{|fuid| {at: Time.now.to_i, diff: 'new friends', friends: [uid, fuid]} }
+    diffs << (f-nf).map{|fuid| {at: Time.now.to_i, diff: 'del friends', friends: [uid, fuid]} }
+    diffs
+end
+
+
+mng=Mongo::MongoClient.new
+wdb = mng['vk_wdog']
+uinfos_col  = wdb['users_info']
+friends_col = wdb['users_info']
+# делаем дело 
+# берём всех засветившихся uid
+uids = Set.new(uinfos_col.distinct('uid')+
+	       friends_col.distinct('uid')+
+	       friends_col.distinct('friends'))
+
+fetch_ships_cmd  = './fetch-ships.rb 2 3122378 | grep -v DEBUG > watchdog.updates'
+uinfos = system( fetch_ships_cmd )# '%s 1 %s |grep -v DEBUG > watchdog.updates'%[fetch_ships_cmd, uids.to_a.join(' ')])
+
+ninfos, nfriends = IO.read('watchdog.updates').split("\n").map{|l| JSON::parse(l)}
+ninfos_col, friends_col = wdb[(uinfos_col.name+'_last')], wdb[(friends_col.name+'_last')]
+ninfos_col   << ninfos
+nfriends_col << nfriends
+nuids = Set.new(ninfos_col.distinct('uid')+
+	       nfriends_col.distinct('uid')+
+	       nfriends_col.distinct('friends'))
+
+nuids.each do |uid|
+	# вычисляем изменения в данных
+    idiffs, fdiffs= info_diff(uid, uinfos[uid], ninfos[uid]), friends_diff(uid, friends[uid], nfriends[uid] )
+    wdb[uinfos_col.name+'_diffs'] << idiffs
+    wdb[friends_col.name+'_diffs'] << fdiffs
+end
+wdb.drop(uinfos_col.name) 
+wdb.drop(friends_col.name)
+wdb.rename(ninfos_col.name, uinfos_col.name)
+wdb.rename(nfriends_col.name, friends_col.name)
+dlog 'Total fetched: infos for %s users, friends for %s'%[ninfos.size, nfriends.size]

data/tests/fetch-ships.rb

@@ -0,0 +1,37 @@
+#!/usr/bin/ruby
+#encoding: utf-8
+require 'bundler/setup'
+require 'vk-watchdog'
+##
+# рекурсивно вытаскиваем инфу и друзей пользователей ВК на заданную глубину
+# 
+
+if ARGV.empty?
+	dlog 'Command: %s depth start_uids 
+Example: fetch-ships.rb 2 1  #=> friends of user 1 and friends of friends of user 1'%[__FILE__]
+	exit 1
+end
+
+depth = ARGV.shift.to_i
+uids = ARGV.map{|a| a.to_i}
+
+dlog 'Fetching vk users infos and friends for depth %s from start uids: %s'%[depth, uids]
+next_uids = uids.dup
+uinfos = {}
+ufriends = {}
+0.upto(depth) do |loop_i|
+	dlog 'Have %s next uids, left %s loops'%[next_uids.size, depth-loop_i]
+	ufriends_uids = next_uids.select{|uid| not ufriends[uid]}
+	ufriends.merge! ZVK::users_friends(ufriends_uids)
+	uinfos_uids = (ufriends.keys+ufriends.values.flatten.uniq)
+#.flatten.flatten.select{|uid| not uinfos[uid]}
+	uinfos.merge! ZVK::users_info(uinfos_uids)
+    	next_uids = ufriends.to_a.flatten.flatten.uniq.select{|uid| not ufriends[uid] and uinfos[uid]}
+
+end
+
+
+puts uinfos.to_json
+puts ufriends.to_json
+
+dlog 'Total fetched: infos for %s users, friends for %s'%[uinfos.size, ufriends.size]

data/tests/watch-ships.rb

@@ -0,0 +1,81 @@
+#!/usr/bin/ruby
+#encoding: utf-8
+require 'bundler/setup'
+require 'vk-ruby'
+require 'mongo'
+require 'set'
+require 'json'
+
+##
+# мониторим наших овечек и записываем все их изменения/обновления:
+# 1. Подтягиваем из монги список друзей, профилей для мониторинга
+# 2.   Для всех получаем актуальные данные, сравниваем - если ==, то пропускаем
+#    если расходятся - тогда обновляем запись в монге и пишем в лог инфу об изменении
+
+def dlog(*args)
+    STDERR.puts args
+end
+
+def info_diff(uid, info, ninfo)
+    diffs=[]
+    Set.new(infos.keys+ninfos.keys).each do |fld|
+    	ofval, nfval = infos[fld], ninfos[fld]
+	next if ofval == nfval
+	if not ofval # значит до обновления филда не было
+		diffs << {at: Time.now.to_i, diff: 'new field', field: {fld=>nfval}}	
+#ninfos_col.update({uid: uid}, {'$set'=>diffs.last}) # и в бд
+	elsif not nfval # а теперь филд удалился
+		diffs << {at: Time.now.to_i, diff: 'del field', field: {fld=>ofval}}
+                #! в базе не оставляем
+#ninfos_col.update({uid: uid}, {'$set'=>diffs.last}) # и в бд
+	elsif nfval != ofval
+		diffs << {at: Time.now.to_i, diff: 'upd field', field: fld, vals: [ofval,nfval]} # филд+старое и новое
+	end
+    end
+    diffs
+end
+
+def friends_diff(uid, friends, nfriends)
+    f, nf = Set.new(friends), Set.new(nfriends)
+    return [] if (f==nf)
+
+    diffs=[]
+# new friends
+    diffs << (nf-f).map{|fuid| {at: Time.now.to_i, diff: 'new friends', friends: [uid, fuid]} }
+    diffs << (f-nf).map{|fuid| {at: Time.now.to_i, diff: 'del friends', friends: [uid, fuid]} }
+    diffs
+end
+
+
+mng=Mongo::MongoClient.new
+wdb = mng['vk_wdog']
+uinfos_col  = wdb['users_info']
+friends_col = wdb['users_info']
+# делаем дело 
+# берём всех засветившихся uid
+uids = Set.new(uinfos_col.distinct('uid')+
+	       friends_col.distinct('uid')+
+	       friends_col.distinct('friends'))
+
+fetch_ships_cmd  = './fetch-ships.rb 2 3122378 | grep -v DEBUG > watchdog.updates'
+uinfos = system( fetch_ships_cmd )# '%s 1 %s |grep -v DEBUG > watchdog.updates'%[fetch_ships_cmd, uids.to_a.join(' ')])
+
+ninfos, nfriends = IO.read('watchdog.updates').split("\n").map{|l| JSON::parse(l)}
+ninfos_col, friends_col = wdb[(uinfos_col.name+'_last')], wdb[(friends_col.name+'_last')]
+ninfos_col   << ninfos
+nfriends_col << nfriends
+nuids = Set.new(ninfos_col.distinct('uid')+
+	       nfriends_col.distinct('uid')+
+	       nfriends_col.distinct('friends'))
+
+nuids.each do |uid|
+	# вычисляем изменения в данных
+    idiffs, fdiffs= info_diff(uid, uinfos[uid], ninfos[uid]), friends_diff(uid, friends[uid], nfriends[uid] )
+    wdb[uinfos_col.name+'_diffs'] << idiffs
+    wdb[friends_col.name+'_diffs'] << fdiffs
+end
+wdb.drop(uinfos_col.name) 
+wdb.drop(friends_col.name)
+wdb.rename(ninfos_col.name, uinfos_col.name)
+wdb.rename(nfriends_col.name, friends_col.name)
+dlog 'Total fetched: infos for %s users, friends for %s'%[ninfos.size, nfriends.size]

metadata

@@ -0,0 +1,52 @@
+--- !ruby/object:Gem::Specification
+name: vk-watchdog
+version: !ruby/object:Gem::Version
+  version: '0.1'
+  prerelease: 
+platform: ruby
+authors:
+- ev42
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-12-23 00:00:00.000000000 Z
+dependencies: []
+description: Простая утилита для мониторинга обновлений профилей ВК (обновлений друзей
+  и инфы).
+email: splin.ev42@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/fetch-ships.rb
+- lib/watch-ships.rb
+- bin/fetch-vk-accs
+- tests/fetch-ships.rb
+- tests/watch-ships.rb
+- Rakefile
+homepage: http://google.com
+licenses:
+- none
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: zlibs::vk
+test_files: []