embulk-input-sitemap 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: f2373cfc6f002bcd47fa75bd7685d7a97a14257e58341d18f1bcc899c96d9626
4
+ data.tar.gz: 216200f9d64b0af61ab84baf15cf9f652bffa58e1789ff3317d3ff54f821b548
5
+ SHA512:
6
+ metadata.gz: b82213ff694d42a2f9df7284222d333ee498704945e01278bdbd91b5f07ea1056c3faa1d9d3a1ca07503c9fde91f2f724900d3cabccab8f8a5f4a96b55e5f54a
7
+ data.tar.gz: 2e20ed289d7aa0e68351951fa88922fa86c1e94b04f9c3d42d364c9fd4a8d631186a5492b0d77cfe5946e54e5a4a7e15fcc23dc5a4076367006e4eae464901cb
data/.gitignore ADDED
@@ -0,0 +1,8 @@
1
+ *~
2
+ /pkg/
3
+ /tmp/
4
+ /.bundle/
5
+ /Gemfile.lock
6
+ /vendor/bundle/
7
+ /workspaces/
8
+ /bin/
data/.ruby-version ADDED
@@ -0,0 +1 @@
1
+ jruby-9.1.13.0
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ source 'https://rubygems.org/'
2
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,21 @@
1
+
2
+ MIT License
3
+
4
+ Permission is hereby granted, free of charge, to any person obtaining
5
+ a copy of this software and associated documentation files (the
6
+ "Software"), to deal in the Software without restriction, including
7
+ without limitation the rights to use, copy, modify, merge, publish,
8
+ distribute, sublicense, and/or sell copies of the Software, and to
9
+ permit persons to whom the Software is furnished to do so, subject to
10
+ the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be
13
+ included in all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,32 @@
1
+ # Sitemap input plugin for Embulk
2
+
3
+ Embulk input plugin for sitemap.xml.
4
+
5
+ ## Overview
6
+
7
+ * **Plugin type**: input
8
+ * **Resume supported**: no
9
+ * **Cleanup supported**: no
10
+ * **Guess supported**: no
11
+
12
+ ## Configuration
13
+
14
+ - **url**: The sitemap url (string, required)
15
+ - **params**: Query parameter for sitemap url (array, default: [])
16
+
17
+ ## Example
18
+
19
+ ```yaml
20
+ in:
21
+ type: sitemap
22
+ url: https://sem-technology.info/sitemap.xml
23
+ params:
24
+ - {name: page, value: 1}
25
+ ```
26
+
27
+
28
+ ## Build
29
+
30
+ ```
31
+ $ rake
32
+ ```
data/Rakefile ADDED
@@ -0,0 +1,3 @@
1
+ require "bundler/gem_tasks"
2
+
3
+ task default: :build
@@ -0,0 +1,21 @@
1
+
2
+ Gem::Specification.new do |spec|
3
+ spec.name = "embulk-input-sitemap"
4
+ spec.version = "0.1.0"
5
+ spec.authors = ["ryota.yamada"]
6
+ spec.summary = "Sitemap input plugin for Embulk"
7
+ spec.description = "Loads records from Sitemap."
8
+ spec.email = ["1987yama3@gmail.com"]
9
+ spec.licenses = ["MIT"]
10
+ spec.homepage = "https://github.com/sem-technology/embulk-input-sitemap"
11
+
12
+ spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
13
+ spec.test_files = spec.files.grep(%r{^(test|spec)/})
14
+ spec.require_paths = ["lib"]
15
+
16
+ spec.add_dependency "nokogiri"
17
+
18
+ spec.add_development_dependency 'embulk', ['>= 0.8.38']
19
+ spec.add_development_dependency 'bundler', ['>= 1.10.6']
20
+ spec.add_development_dependency 'rake', ['>= 10.0']
21
+ end
@@ -0,0 +1,5 @@
1
+ in:
2
+ type: sitemap
3
+ url: https://sem-technology.info/sitemap.xml
4
+ out:
5
+ type: stdout
@@ -0,0 +1,9 @@
1
+ ENV["SSL_CERT_FILE"] = "/Users/yamada/programing/embulk/embulk-input-sitemap/bin/cacert.pem"
2
+ require "embulk/input/sitemap/plugin"
3
+ require "embulk/input/sitemap/client"
4
+ module Embulk
5
+ module Input
6
+ module Sitemap
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,56 @@
1
+ require "ostruct"
2
+ require "open-uri"
3
+ require "nokogiri"
4
+ require "zlib"
5
+ module Embulk
6
+ module Input
7
+ module Sitemap
8
+ class Client
9
+ def initialize(url, params)
10
+ @url = url
11
+ @params = params
12
+ end
13
+
14
+ def query
15
+ queries = @params.map{|param| "#{param["name"]}=#{param["value"]}"}
16
+ return "?" + queries.join("&") if queries.length > 0 and @url.include?("?") == false
17
+ return "&" + queries.join("&") if queries.length > 0 and @url.include?("?")
18
+ return ""
19
+ end
20
+
21
+ def invoke
22
+ items = []
23
+ Embulk.logger.info "GET #{@url}#{self.query}"
24
+ response = open(@url + self.query) do |f|
25
+ Embulk.logger.info "Content-Type = #{f.content_type}"
26
+ case f.content_type
27
+ when "application/x-gzip", "application/octet-stream" then
28
+ `curl #{@url} | gunzip -d`
29
+ else f.read
30
+ end
31
+ end
32
+ document = Nokogiri::XML(response)
33
+ sitemaps = document.css("sitemap")
34
+ urls = document.css("url")
35
+ Embulk.logger.info "Find #{sitemaps.length} sitemaps, #{urls.length} urls"
36
+ if sitemaps.length > 0
37
+ items << sitemaps.collect do |sitemap|
38
+ Client.new(sitemap.css("loc").first.text.to_s).invoke
39
+ end
40
+ end
41
+ if urls.length > 0
42
+ items << urls.collect do |url|
43
+ item = {}
44
+ item[:loc] = url.css("loc").first.text.to_s
45
+ item[:changefreq] = url.css("changefreq").first.text.to_s if url.css("changefreq").first
46
+ item[:priority] = url.css("priority").first.text.to_s if url.css("priority").first
47
+ item[:lastmod] = url.css("lastmod").first.text.to_s if url.css("lastmod").first
48
+ OpenStruct.new(item)
49
+ end
50
+ end
51
+ items.flatten
52
+ end
53
+ end
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,52 @@
1
+ module Embulk
2
+ module Input
3
+ module Sitemap
4
+ class Plugin < InputPlugin
5
+ Embulk::Plugin.register_input("sitemap", self)
6
+
7
+ def self.transaction(config, &control)
8
+ # configuration code:
9
+ task = {
10
+ "url" => config.param("url", :string),
11
+ "params" => config.param("params", :array, default: []),
12
+ }
13
+
14
+ columns = [
15
+ Column.new(0, "loc", :string),
16
+ Column.new(1, "lastmod", :string),
17
+ Column.new(2, "priority", :double),
18
+ Column.new(3, "changefreq", :string),
19
+ ]
20
+
21
+ resume(task, columns, 1, &control)
22
+ end
23
+
24
+ def self.resume(task, columns, count, &control)
25
+ task_reports = yield(task, columns, count)
26
+
27
+ next_config_diff = {}
28
+ return next_config_diff
29
+ end
30
+
31
+ def init
32
+ end
33
+
34
+ def run
35
+ client = Embulk::Input::Sitemap::Client.new(task["url"], task["params"])
36
+ client.invoke.each do |item|
37
+ page_builder.add([
38
+ item.loc,
39
+ item.lastmod,
40
+ item.priority,
41
+ item.changefreq,
42
+ ])
43
+ end
44
+ page_builder.finish
45
+
46
+ task_report = {}
47
+ return task_report
48
+ end
49
+ end
50
+ end
51
+ end
52
+ end
metadata ADDED
@@ -0,0 +1,111 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: embulk-input-sitemap
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - ryota.yamada
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2017-12-12 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - ">="
17
+ - !ruby/object:Gem::Version
18
+ version: '0'
19
+ name: nokogiri
20
+ prerelease: false
21
+ type: :runtime
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ requirement: !ruby/object:Gem::Requirement
29
+ requirements:
30
+ - - ">="
31
+ - !ruby/object:Gem::Version
32
+ version: 0.8.38
33
+ name: embulk
34
+ prerelease: false
35
+ type: :development
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: 0.8.38
41
+ - !ruby/object:Gem::Dependency
42
+ requirement: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - ">="
45
+ - !ruby/object:Gem::Version
46
+ version: 1.10.6
47
+ name: bundler
48
+ prerelease: false
49
+ type: :development
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: 1.10.6
55
+ - !ruby/object:Gem::Dependency
56
+ requirement: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - ">="
59
+ - !ruby/object:Gem::Version
60
+ version: '10.0'
61
+ name: rake
62
+ prerelease: false
63
+ type: :development
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '10.0'
69
+ description: Loads records from Sitemap.
70
+ email:
71
+ - 1987yama3@gmail.com
72
+ executables: []
73
+ extensions: []
74
+ extra_rdoc_files: []
75
+ files:
76
+ - ".gitignore"
77
+ - ".ruby-version"
78
+ - Gemfile
79
+ - LICENSE.txt
80
+ - README.md
81
+ - Rakefile
82
+ - embulk-input-sitemap.gemspec
83
+ - examples/normal.yml
84
+ - lib/embulk/input/sitemap.rb
85
+ - lib/embulk/input/sitemap/client.rb
86
+ - lib/embulk/input/sitemap/plugin.rb
87
+ homepage: https://github.com/sem-technology/embulk-input-sitemap
88
+ licenses:
89
+ - MIT
90
+ metadata: {}
91
+ post_install_message:
92
+ rdoc_options: []
93
+ require_paths:
94
+ - lib
95
+ required_ruby_version: !ruby/object:Gem::Requirement
96
+ requirements:
97
+ - - ">="
98
+ - !ruby/object:Gem::Version
99
+ version: '0'
100
+ required_rubygems_version: !ruby/object:Gem::Requirement
101
+ requirements:
102
+ - - ">="
103
+ - !ruby/object:Gem::Version
104
+ version: '0'
105
+ requirements: []
106
+ rubyforge_project:
107
+ rubygems_version: 2.6.13
108
+ signing_key:
109
+ specification_version: 4
110
+ summary: Sitemap input plugin for Embulk
111
+ test_files: []