vortex_client 0.6.0 → 0.7.0
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/examples/README.rdoc +30 -0
- data/examples/create_personpresentations.rb +147 -0
- data/examples/dice.gif +0 -0
- data/examples/import_static_site.rb +211 -0
- data/examples/no_right_margin.rb +11 -0
- data/examples/propfind_proppatch.rb +7 -0
- data/examples/replace_spike.rb +42 -0
- data/examples/scrape_hero_publications.rb +38 -0
- data/examples/scrape_holocaust.rb +351 -0
- data/examples/scrape_holocaust_related_links.rb +134 -0
- data/examples/scrape_vortex_search.rb +44 -0
- data/examples/search_replace_documents.rb +57 -0
- data/examples/search_replace_documents_frontpage.rb +88 -0
- data/examples/test_searc_replace.rb +3 -0
- data/examples/unpublish.rb +11 -0
- data/examples/upload_image.rb +8 -0
- data/examples/upload_image_flymake.rb +8 -0
- data/lib/vortex_client.rb +2 -1
- data/test/test_vortex_article_publish.rb +3 -16
- metadata +48 -17
- data/.gitignore +0 -21
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
require 'rubygems'
|
|
2
|
+
require 'open-uri'
|
|
3
|
+
require 'nokogiri'
|
|
4
|
+
require 'cgi'
|
|
5
|
+
require 'pp'
|
|
6
|
+
require 'uri'
|
|
7
|
+
|
|
8
|
+
# Search a vortex host for a query string by scraping search results
|
|
9
|
+
# Regexp filter for url's is optional.
|
|
10
|
+
#
|
|
11
|
+
# Usage:
|
|
12
|
+
#
|
|
13
|
+
# result = vortex_search("www.hf.uio.no", "Ansettelsesforhold ved UiO")
|
|
14
|
+
# => ["http://www.hf.uio.no/ifikk/for-ansatte/ansettelsesforhold/index.html",
|
|
15
|
+
# "http://www.hf.uio.no/imv/for-ansatte/ansettelsesforhold/",
|
|
16
|
+
# "http://www.hf.uio.no/iln/for-ansatte/ansettelsesforhold/"]
|
|
17
|
+
def vortex_search(host, query)
|
|
18
|
+
uri = URI.parse(host)
|
|
19
|
+
if(uri.host)then
|
|
20
|
+
host = uri.host
|
|
21
|
+
end
|
|
22
|
+
q = CGI::escape(query)
|
|
23
|
+
search_results = {}
|
|
24
|
+
prev_page = nil
|
|
25
|
+
current_page = ""
|
|
26
|
+
i = 1
|
|
27
|
+
# Manage paging of search results
|
|
28
|
+
while(prev_page != current_page)do
|
|
29
|
+
prev_page = current_page
|
|
30
|
+
url = "http://#{host}/?vrtx=search&page=#{i}&query=#{q}"
|
|
31
|
+
doc = Nokogiri::HTML(open(url))
|
|
32
|
+
doc.css('span.url').each do |link|
|
|
33
|
+
search_results[link.content] = true
|
|
34
|
+
end
|
|
35
|
+
current_page = doc.to_s
|
|
36
|
+
i = i + 1
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
result = []
|
|
40
|
+
search_results.keys.each do |link|
|
|
41
|
+
result << link
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# Search and replace a set of sentences in a set of folders on vortex webdav server.
|
|
2
|
+
#
|
|
3
|
+
# TODO: Make the script keep the lastUpdatedDate and publishedDate unchanged.
|
|
4
|
+
#
|
|
5
|
+
# Author: Thomas Flemming 2010 thomasfl@usit.uio.no
|
|
6
|
+
|
|
7
|
+
require 'rubygems'
|
|
8
|
+
require 'vortex_client'
|
|
9
|
+
require 'json'
|
|
10
|
+
|
|
11
|
+
replacements = {
|
|
12
|
+
"<h2>Teaching</h2>" => "<h2>Courses taught</h2>",
|
|
13
|
+
"<h2>Higher education and employment history</h2>" => "<h2>Background</h2>",
|
|
14
|
+
"<h2>Honoraria</h2>" => "<h2>Awards</h2>",
|
|
15
|
+
"<h2>Appointments</h2>" => "<h2>Positions held</h2>",
|
|
16
|
+
"<h2>Cooperation</h2>" => "<h2>Partners</h2>"
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
folders = [
|
|
20
|
+
"/isv/english/people/aca/",
|
|
21
|
+
"/iss/english/people/aca/",
|
|
22
|
+
"/psi/english/people/aca/",
|
|
23
|
+
"/sai/english/people/aca/",
|
|
24
|
+
"/econ/english/people/aca/",
|
|
25
|
+
"/arena/english/people/aca/",
|
|
26
|
+
"/tik/english/people/aca/"
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
vortex = Vortex::Connection.new("https://nyweb1-dav.uio.no/",:use_osx_keychain => true)
|
|
31
|
+
|
|
32
|
+
folders.each do |folder|
|
|
33
|
+
|
|
34
|
+
vortex.find(folder,:recursive => true,:filename=>/\.html$/) do |item|
|
|
35
|
+
puts item.uri.to_s
|
|
36
|
+
|
|
37
|
+
data = nil
|
|
38
|
+
begin
|
|
39
|
+
data = JSON.parse(item.content)
|
|
40
|
+
rescue
|
|
41
|
+
puts "Warning. Bad document. Not json:" + item.uri.to_s
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
if(data)then
|
|
45
|
+
content = data["properties"]["content"]
|
|
46
|
+
if(content)then
|
|
47
|
+
replacements.each do |key,val|
|
|
48
|
+
content = content.gsub(key,val)
|
|
49
|
+
end
|
|
50
|
+
data["properties"]["content"] = content
|
|
51
|
+
item.content = data.to_json
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
end
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# Search and replace a set of sentences in a set of folders on vortex webdav server.
|
|
3
|
+
#
|
|
4
|
+
#
|
|
5
|
+
# for alle forside-dokumenter på jus.uio.no, tf.uio.no, hf.uio.no, uv.uio.no, sv.uio.no, odontologi.uio.no
|
|
6
|
+
#
|
|
7
|
+
#
|
|
8
|
+
# Author: Thomas Flemming 2010 thomasfl@usit.uio.no
|
|
9
|
+
|
|
10
|
+
require 'rubygems'
|
|
11
|
+
require 'vortex_client'
|
|
12
|
+
require 'json'
|
|
13
|
+
require 'scrape_vortex_search'
|
|
14
|
+
require 'pp'
|
|
15
|
+
require 'uri'
|
|
16
|
+
|
|
17
|
+
replacements = {
|
|
18
|
+
"Ansettelsesforhold ved UiO" => "Ansettelsesforhold for alle ansatte ved UiO",
|
|
19
|
+
"Arbeidsstøtte ved UiO" => "Arbeidsstøtte for alle ansatte ved UiO",
|
|
20
|
+
"Drift og servicefunksjoner ved UiO" =>"Drift og servicefunksjoner for alle ansatte ved UiO",
|
|
21
|
+
"Kompetanseutvikling ved UiO" => "Kompetanseutvikling for alle ansatte ved UiO"
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
hosts = [
|
|
25
|
+
"https://www-dav.hf.uio.no/",
|
|
26
|
+
"https://www-dav.odont.uio.no",
|
|
27
|
+
"https://www-dav.jus.uio.no/",
|
|
28
|
+
"https://www-dav.tf.uio.no/",
|
|
29
|
+
"https://www-dav.uv.uio.no/",
|
|
30
|
+
"https://www-dav.sv.uio.no/"
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
# Opens a json document in vortex.
|
|
34
|
+
#
|
|
35
|
+
# Example:
|
|
36
|
+
#
|
|
37
|
+
# with_json_doc(path) do |dav_item, json_data|
|
|
38
|
+
# puts json_data["resourcetype"]
|
|
39
|
+
# dav_item.content = dav_item.content.sub('Oxford','UiO')
|
|
40
|
+
# end
|
|
41
|
+
|
|
42
|
+
def with_json_doc(url)
|
|
43
|
+
vortex = Vortex::Connection.new(url,:use_osx_keychain => true)
|
|
44
|
+
if(not(vortex.exists?(url)))then
|
|
45
|
+
puts "Warning: Can't find " + url
|
|
46
|
+
return -1
|
|
47
|
+
end
|
|
48
|
+
vortex.find(url) do |item|
|
|
49
|
+
begin
|
|
50
|
+
data = JSON.parse(item.content)
|
|
51
|
+
yield item, data
|
|
52
|
+
rescue
|
|
53
|
+
return -1
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
hosts.each do |host|
|
|
59
|
+
search_host = host.sub("www-dav","www")
|
|
60
|
+
replacements.each do |from, to|
|
|
61
|
+
puts "Replacing: '#{from}' => '#{to}'"
|
|
62
|
+
search_results = vortex_search(search_host, from)
|
|
63
|
+
# TODO remove not( /(\.html)|\// alt untatt '.../' eller 'html'
|
|
64
|
+
search_results.delete_if {|x| x =~ /\.pdf$/} # Remove pdf files
|
|
65
|
+
count = 0
|
|
66
|
+
search_results.each do |path|
|
|
67
|
+
if(not(path =~ /\.html$/))then
|
|
68
|
+
path = path + "index.html"
|
|
69
|
+
end
|
|
70
|
+
# count = count + replace(host, path, from, to, "frontpage")
|
|
71
|
+
path = path.sub(/^http:/, 'https:').sub(/\/\/www\./,'//www-dav.')
|
|
72
|
+
with_json_doc(path) do |item, data|
|
|
73
|
+
if(data["resourcetype"] == "frontpage")then
|
|
74
|
+
puts "Updating:" + path
|
|
75
|
+
item.content = item.content.sub(from,to)
|
|
76
|
+
|
|
77
|
+
if(item.content.to_s =~ Regexp.new(from))then
|
|
78
|
+
puts "Warning: Not updated."
|
|
79
|
+
else
|
|
80
|
+
count = count + 1
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
puts "Updated #{count.to_s} documents."
|
|
86
|
+
puts
|
|
87
|
+
end
|
|
88
|
+
end
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
require 'rubygems'
|
|
2
|
+
require 'vortex_client'
|
|
3
|
+
|
|
4
|
+
# Hides documents for visitors by setting the documents unpublish-date to current time and date.
|
|
5
|
+
#
|
|
6
|
+
# Find all files recursively under '/people' with name 'index.html', and set "unpublished-date" to now.
|
|
7
|
+
|
|
8
|
+
@vortex = Vortex::Connection.new("https://nyweb2-dav.uio.no/", :use_osx_keychain => true)
|
|
9
|
+
@vortex.find('/people/', :recursive => true, :filename=>/\.html$/) do |item|
|
|
10
|
+
item.proppatch('<v:unpublish-date xmlns:v="vrtx">'+Time.now.httpdate.to_s+'</v:unpublish-date>')
|
|
11
|
+
end
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
require 'rubygems'
|
|
2
|
+
require 'vortex_client'
|
|
3
|
+
|
|
4
|
+
# Upload a binary file to Vortex
|
|
5
|
+
vortex = Vortex::Connection.new("https://vortex-dav.uio.no/")
|
|
6
|
+
vortex.cd('/brukere/thomasfl/')
|
|
7
|
+
content = open("dice.gif", "rb") {|io| io.read }
|
|
8
|
+
vortex.put_string("dice_6.gif", content)
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
require 'rubygems'
|
|
2
|
+
require 'vortex_client'
|
|
3
|
+
|
|
4
|
+
# Upload a binary file to Vortex
|
|
5
|
+
vortex = Vortex::Connection.new("https://vortex-dav.uio.no/")
|
|
6
|
+
vortex.cd('/brukere/thomasfl/')
|
|
7
|
+
content = open("dice.gif", "rb") {|io| io.read }
|
|
8
|
+
vortex.put_string("dice_6.gif", content)
|
data/lib/vortex_client.rb
CHANGED
|
@@ -52,7 +52,7 @@ module Vortex
|
|
|
52
52
|
@handler = NetHttpHandler.new(@uri)
|
|
53
53
|
@handler.verify_server = false # This defaults to true in Net::DAV
|
|
54
54
|
if(args != [])
|
|
55
|
-
if(args[0][:use_osx_keychain])then
|
|
55
|
+
if(args[0][:use_osx_keychain] or args[0][:osx_keychain])then
|
|
56
56
|
|
|
57
57
|
# Retrieve password from OS X KeyChain.
|
|
58
58
|
osx = (RUBY_PLATFORM =~ /darwin/)
|
|
@@ -116,6 +116,7 @@ module Vortex
|
|
|
116
116
|
uri = @uri.merge(object.url)
|
|
117
117
|
# puts "DEBUG: uri = " + uri.to_s
|
|
118
118
|
self.put_string(uri, object.content)
|
|
119
|
+
# puts "DEBUG: object.properties: #{uri}\n#{object.properties.gsub("><",">\n<")}\n-----\n"
|
|
119
120
|
self.proppatch(uri, object.properties)
|
|
120
121
|
return uri.to_s
|
|
121
122
|
else
|
|
@@ -6,10 +6,9 @@ class TestVortexClientUtils < Test::Unit::TestCase
|
|
|
6
6
|
|
|
7
7
|
def setup
|
|
8
8
|
if(not(@vortex))
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
@vortex = Connection.new("https://vortex-dav.uio.no/", :use_osx_keychain => true)
|
|
9
|
+
user = ENV['DAVUSER']
|
|
10
|
+
pass = ENV['DAVPASS']
|
|
11
|
+
@vortex = Connection.new("https://vortex-dav.uio.no/",user, pass)
|
|
13
12
|
end
|
|
14
13
|
end
|
|
15
14
|
|
|
@@ -23,18 +22,6 @@ class TestVortexClientUtils < Test::Unit::TestCase
|
|
|
23
22
|
article = Vortex::HtmlArticle.new(:title => "Sample Title 2",
|
|
24
23
|
:introduction => "Sample introduction",
|
|
25
24
|
:body => "<p>Hello world</p>",
|
|
26
|
-
:tags => ["Tag1", "Tag2"],
|
|
27
|
-
## :date => Time.now,
|
|
28
|
-
:publishedDate => "05.01.2010 12:00",
|
|
29
|
-
:author => "Thomas Flemming")
|
|
30
|
-
|
|
31
|
-
@vortex.publish(article)
|
|
32
|
-
assert @vortex.exists?(url)
|
|
33
|
-
|
|
34
|
-
article = Vortex::StrucHtmlArticle.new(:title => "Sample Title 2",
|
|
35
|
-
:introduction => "Sample introduction",
|
|
36
|
-
:body => "<p>Hello world</p>",
|
|
37
|
-
:tags => ["Tag1", "Tag2"],
|
|
38
25
|
## :date => Time.now,
|
|
39
26
|
:publishedDate => "05.01.2010 12:00",
|
|
40
27
|
:author => "Thomas Flemming")
|
metadata
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: vortex_client
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
hash:
|
|
5
|
-
prerelease:
|
|
4
|
+
hash: 3
|
|
5
|
+
prerelease:
|
|
6
6
|
segments:
|
|
7
7
|
- 0
|
|
8
|
-
-
|
|
8
|
+
- 7
|
|
9
9
|
- 0
|
|
10
|
-
version: 0.
|
|
10
|
+
version: 0.7.0
|
|
11
11
|
platform: ruby
|
|
12
12
|
authors:
|
|
13
13
|
- Thomas Flemming
|
|
@@ -15,7 +15,7 @@ autorequire:
|
|
|
15
15
|
bindir: bin
|
|
16
16
|
cert_chain: []
|
|
17
17
|
|
|
18
|
-
date:
|
|
18
|
+
date: 2011-05-30 00:00:00 +02:00
|
|
19
19
|
default_executable: vrtx-sync
|
|
20
20
|
dependencies:
|
|
21
21
|
- !ruby/object:Gem::Dependency
|
|
@@ -75,20 +75,32 @@ extra_rdoc_files:
|
|
|
75
75
|
- README.rdoc
|
|
76
76
|
files:
|
|
77
77
|
- .document
|
|
78
|
-
- .gitignore
|
|
79
78
|
- LICENSE
|
|
80
79
|
- README.rdoc
|
|
81
80
|
- Rakefile
|
|
82
81
|
- VERSION
|
|
83
82
|
- bin/vrtx-sync
|
|
83
|
+
- examples/README.rdoc
|
|
84
84
|
- examples/change_folder_type.rb
|
|
85
85
|
- examples/create_collection.rb
|
|
86
|
+
- examples/create_personpresentations.rb
|
|
87
|
+
- examples/dice.gif
|
|
86
88
|
- examples/disable_right_column.rb
|
|
89
|
+
- examples/import_static_site.rb
|
|
87
90
|
- examples/ldap_util.rb
|
|
91
|
+
- examples/no_right_margin.rb
|
|
88
92
|
- examples/person_presentation.rb
|
|
93
|
+
- examples/propfind_proppatch.rb
|
|
89
94
|
- examples/publish_article.rb
|
|
90
95
|
- examples/publish_event.rb
|
|
96
|
+
- examples/scrape_hero_publications.rb
|
|
97
|
+
- examples/scrape_holocaust.rb
|
|
98
|
+
- examples/scrape_holocaust_related_links.rb
|
|
99
|
+
- examples/scrape_vortex_search.rb
|
|
100
|
+
- examples/search_replace_documents.rb
|
|
91
101
|
- examples/sitemap.rb
|
|
102
|
+
- examples/unpublish.rb
|
|
103
|
+
- examples/upload_image.rb
|
|
92
104
|
- lib/vortex_client.rb
|
|
93
105
|
- lib/vortex_client/item_extensions.rb
|
|
94
106
|
- lib/vortex_client/person.rb
|
|
@@ -105,13 +117,17 @@ files:
|
|
|
105
117
|
- test/test_vortex_pic.rb
|
|
106
118
|
- test/test_vortex_tags.rb
|
|
107
119
|
- test/test_vortex_utils.rb
|
|
120
|
+
- examples/replace_spike.rb
|
|
121
|
+
- examples/search_replace_documents_frontpage.rb
|
|
122
|
+
- examples/test_searc_replace.rb
|
|
123
|
+
- examples/upload_image_flymake.rb
|
|
108
124
|
has_rdoc: true
|
|
109
125
|
homepage: http://github.com/thomasfl/vortex_client
|
|
110
126
|
licenses: []
|
|
111
127
|
|
|
112
128
|
post_install_message:
|
|
113
|
-
rdoc_options:
|
|
114
|
-
|
|
129
|
+
rdoc_options: []
|
|
130
|
+
|
|
115
131
|
require_paths:
|
|
116
132
|
- lib
|
|
117
133
|
required_ruby_version: !ruby/object:Gem::Requirement
|
|
@@ -135,11 +151,34 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
135
151
|
requirements: []
|
|
136
152
|
|
|
137
153
|
rubyforge_project:
|
|
138
|
-
rubygems_version: 1.
|
|
154
|
+
rubygems_version: 1.5.0
|
|
139
155
|
signing_key:
|
|
140
156
|
specification_version: 3
|
|
141
157
|
summary: Vortex CMS client
|
|
142
158
|
test_files:
|
|
159
|
+
- examples/change_folder_type.rb
|
|
160
|
+
- examples/create_collection.rb
|
|
161
|
+
- examples/create_personpresentations.rb
|
|
162
|
+
- examples/disable_right_column.rb
|
|
163
|
+
- examples/import_static_site.rb
|
|
164
|
+
- examples/ldap_util.rb
|
|
165
|
+
- examples/no_right_margin.rb
|
|
166
|
+
- examples/person_presentation.rb
|
|
167
|
+
- examples/propfind_proppatch.rb
|
|
168
|
+
- examples/publish_article.rb
|
|
169
|
+
- examples/publish_event.rb
|
|
170
|
+
- examples/replace_spike.rb
|
|
171
|
+
- examples/scrape_hero_publications.rb
|
|
172
|
+
- examples/scrape_holocaust.rb
|
|
173
|
+
- examples/scrape_holocaust_related_links.rb
|
|
174
|
+
- examples/scrape_vortex_search.rb
|
|
175
|
+
- examples/search_replace_documents.rb
|
|
176
|
+
- examples/search_replace_documents_frontpage.rb
|
|
177
|
+
- examples/sitemap.rb
|
|
178
|
+
- examples/test_searc_replace.rb
|
|
179
|
+
- examples/unpublish.rb
|
|
180
|
+
- examples/upload_image.rb
|
|
181
|
+
- examples/upload_image_flymake.rb
|
|
143
182
|
- test/helper.rb
|
|
144
183
|
- test/test_acl.rb
|
|
145
184
|
- test/test_date_conversion.rb
|
|
@@ -152,11 +191,3 @@ test_files:
|
|
|
152
191
|
- test/test_vortex_pic.rb
|
|
153
192
|
- test/test_vortex_tags.rb
|
|
154
193
|
- test/test_vortex_utils.rb
|
|
155
|
-
- examples/change_folder_type.rb
|
|
156
|
-
- examples/create_collection.rb
|
|
157
|
-
- examples/disable_right_column.rb
|
|
158
|
-
- examples/ldap_util.rb
|
|
159
|
-
- examples/person_presentation.rb
|
|
160
|
-
- examples/publish_article.rb
|
|
161
|
-
- examples/publish_event.rb
|
|
162
|
-
- examples/sitemap.rb
|