embulk-filter-google_vision_api 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 129cfa811e9b9b1a2e94004d2c32fc8c7008cc99
4
+ data.tar.gz: be9ea20997ac41ee0126b5b137c175defbeb2e7d
5
+ SHA512:
6
+ metadata.gz: 0177f5ffe7ae5f07012ae1a934ba26b2bcbf87ef4dc3ea24946af7a6f3c33a1458fdde84e6066937ec5dde9d5f49c87e74d1b59aab1c1bd6ebebe17897c469f1
7
+ data.tar.gz: a11a2816eea10df168d759783f5c9d5b2b66071b178cb540bc153b870ccd777ce94f07d228ea63ad427e72aec3e162623da35b155bf9ad5d8631fdb3e1744b91
@@ -0,0 +1,5 @@
1
+ *~
2
+ /pkg/
3
+ /tmp/
4
+ /.bundle/
5
+ /Gemfile.lock
@@ -0,0 +1 @@
1
+ jruby-9.1.5.0
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ source 'https://rubygems.org/'
2
+ gemspec
@@ -0,0 +1,21 @@
1
+
2
+ MIT License
3
+
4
+ Permission is hereby granted, free of charge, to any person obtaining
5
+ a copy of this software and associated documentation files (the
6
+ "Software"), to deal in the Software without restriction, including
7
+ without limitation the rights to use, copy, modify, merge, publish,
8
+ distribute, sublicense, and/or sell copies of the Software, and to
9
+ permit persons to whom the Software is furnished to do so, subject to
10
+ the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be
13
+ included in all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,73 @@
1
+ # Google Vision Api filter plugin for Embulk
2
+
3
+ Google Vision Api filter plugin for Embulk.
4
+
5
+ Very easy image recognition.
6
+
7
+ ## Overview
8
+
9
+ * **Plugin type**: filter
10
+
11
+ ## Configuration
12
+
13
+ - **out_key_name**: out_key_name (string)
14
+ - **image_path_key_name**: image_path_key_name (string)
15
+ - **features**: features (array)
16
+ - **type**: FACE_DETECTION or LANDMARK_DETECTION or LOGO_DETECTION or LABEL_DETECTION or TEXT_DETECTION or SAFE_SEARCH_DETECTION or IMAGE_PROPERTIES (string)
17
+ - **maxResults**: maxResults (integer)
18
+ - **delay**: delay (integer, default: 0)
19
+ - **image_num_per_request**: image_num_per_request (integer, default: 16)
20
+ - **google_api_key**: google_api_key (string, default: ENV['GOOGLE_API_KEY'])
21
+
22
+ ## Example
23
+
24
+ ### input
25
+ ```yaml
26
+ - { image_path: 'http://www.embulk.org/docs/_images/embulk-logo.png' }
27
+ ```
28
+
29
+ * respond localfile path and http URI.
30
+
31
+ ### setting
32
+ ```yaml
33
+ filters:
34
+ - type: google_vision_api
35
+ image_path_key_name: image_path
36
+ out_key_name: image_info
37
+ image_num_per_request: 5
38
+ features:
39
+ - {type: FACE_DETECTION, "maxResults":5 }
40
+ - {type: LANDMARK_DETECTION, "maxResults":5 }
41
+ - {type: LOGO_DETECTION, "maxResults":5 }
42
+ - {type: LABEL_DETECTION, "maxResults":5 }
43
+ - {type: TEXT_DETECTION, "maxResults":5 }
44
+ - {type: SAFE_SEARCH_DETECTION, "maxResults":5 }
45
+ - {type: IMAGE_PROPERTIES, "maxResults":5 }
46
+ ```
47
+
48
+ ### output
49
+ ```
50
+ image_path (string) : http://www.embulk.org/docs/_images/embulk-logo.png
51
+ image_info ( json) : {"labelAnnotations":[{"mid":"/m/0dwx7","description":"logo","score":0.86478204},{"mid":"/m/03gq5hm","description":"font","score":0.8472268},{"mid":"/m/0215n","description":"cartoon","score":0.82311255},{"mid":"/m/03g09t","description":"clip art","score":0.69382942},{"mid":"/m/01cd9","description":"brand","score":0.59691668}],"textAnnotations":[{"locale":"id","description":"embulk\n","boundingPoly":{"vertices":[{"x":67,"y":475},{"x":812,"y":475},{"x":812,"y":629},{"x":67,"y":629}]}},{"description":"embulk","boundingPoly":{"vertices":[{"x":68,"y":475},{"x":813,"y":475},{"x":813,"y":629},{"x":68,"y":629}]}}],"safeSearchAnnotation":{"adult":"VERY_UNLIKELY","spoof":"UNLIKELY","medical":"UNLIKELY","violence":"VERY_UNLIKELY"},"imagePropertiesAnnotation":{"dominantColors":{"colors":[{"color":{"red":231,"green":59,"blue":11},"score":0.26240975,"pixelFraction":0.029842343},{"color":{"red":243,"green":176,"blue":124},"score":0.0057866224,"pixelFraction":0.010698198},{"color":{"red":252,"green":250,"blue":248},"score":0.0025187095,"pixelFraction":0.6255005},{"color":{"red":232,"green":79,"blue":16},"score":0.17388013,"pixelFraction":0.023773775},{"color":{"red":225,"green":75,"blue":27},"score":0.14469221,"pixelFraction":0.034909911},{"color":{"red":222,"green":52,"blue":17},"score":0.13995738,"pixelFraction":0.018768769},{"color":{"red":243,"green":106,"blue":31},"score":0.090499125,"pixelFraction":0.025525525},{"color":{"red":243,"green":128,"blue":38},"score":0.064238794,"pixelFraction":0.050425425},{"color":{"red":242,"green":104,"blue":47},"score":0.033798043,"pixelFraction":0.014451952},{"color":{"red":238,"green":131,"blue":65},"score":0.027836611,"pixelFraction":0.010948448}]}}}
52
+ ```
53
+
54
+ ## Vision API Limits
55
+
56
+ | Type of Limit | Usage Limit |
57
+ |:-----------|------------:|
58
+ | MB per image | 4 MB |
59
+ | MB per request | 8 MB |
60
+ | Requests per second | 10 |
61
+ | Requests per feature per day | 700,000 |
62
+ | Requests per feature per month | 20,000,000 |
63
+ | Images per second | 8 |
64
+ | Images per request | 16 |
65
+
66
+ see. [Usage Limits  \|  Google Cloud Vision API  \|  Google Cloud Platform](https://cloud.google.com/vision/limits)
67
+
68
+
69
+ ## Build
70
+
71
+ ```
72
+ $ rake
73
+ ```
@@ -0,0 +1,3 @@
1
+ require "bundler/gem_tasks"
2
+
3
+ task default: :build
@@ -0,0 +1,20 @@
1
+
2
+ Gem::Specification.new do |spec|
3
+ spec.name = "embulk-filter-google_vision_api"
4
+ spec.version = "0.1.0"
5
+ spec.authors = ["toyama0919"]
6
+ spec.summary = "Google Vision Api filter plugin for Embulk"
7
+ spec.description = "Google Vision Api filter plugin for Embulk. Very easy image recognition."
8
+ spec.email = ["toyama0919@gmail.com"]
9
+ spec.licenses = ["MIT"]
10
+ spec.homepage = "https://github.com/toyama0919/embulk-filter-google_vision_api"
11
+
12
+ spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
13
+ spec.test_files = spec.files.grep(%r{^(test|spec)/})
14
+ spec.require_paths = ["lib"]
15
+
16
+ spec.add_dependency 'jruby-openssl'
17
+ spec.add_development_dependency 'embulk', ['>= 0.8.15']
18
+ spec.add_development_dependency 'bundler', ['>= 1.10.6']
19
+ spec.add_development_dependency 'rake', ['>= 10.0']
20
+ end
@@ -0,0 +1,102 @@
1
+ require "json"
2
+ require "net/http"
3
+ require "uri"
4
+ require "openssl"
5
+ require "base64"
6
+ require "pp"
7
+
8
+ module Embulk
9
+ module Filter
10
+
11
+ class GoogleVisionApi < FilterPlugin
12
+ Plugin.register_filter("google_vision_api", self)
13
+ ENDPOINT_PREFIX = "https://vision.googleapis.com/v1/images:annotate"
14
+
15
+ def self.transaction(config, in_schema, &control)
16
+ task = {
17
+ "out_key_name" => config.param("out_key_name", :string),
18
+ "image_path_key_name" => config.param("image_path_key_name", :string),
19
+ "features" => config.param("features", :array),
20
+ "delay" => config.param("delay", :integer, default: 0),
21
+ "image_num_per_request" => config.param("image_num_per_request", :integer, default: 16),
22
+ "google_api_key" => config.param("google_api_key", :string, default: ENV['GOOGLE_API_KEY']),
23
+ }
24
+
25
+ add_columns = [
26
+ Column.new(nil, task["out_key_name"], :json)
27
+ ]
28
+
29
+ out_columns = in_schema + add_columns
30
+
31
+ yield(task, out_columns)
32
+ end
33
+
34
+ def init
35
+ @uri = URI.parse("#{ENDPOINT_PREFIX}?key=#{task['google_api_key']}")
36
+ @http = Net::HTTP.new(@uri.host, @uri.port)
37
+ @http.use_ssl = true
38
+ @http.verify_mode = OpenSSL::SSL::VERIFY_NONE
39
+ @post = Net::HTTP::Post.new(@uri.request_uri, initheader = {'Content-Type' =>'application/json'})
40
+ @image_path_key_name = task['image_path_key_name']
41
+ @out_key_name = task['out_key_name']
42
+ @delay = task['delay']
43
+ @image_num_per_request = task['image_num_per_request']
44
+ @features = task['features']
45
+ end
46
+
47
+ def close
48
+ end
49
+
50
+ def add(page)
51
+ record_groups = page.map { |record|
52
+ Hash[in_schema.names.zip(record)]
53
+ }.each_slice(@image_num_per_request).to_a
54
+
55
+ record_groups.each do |records|
56
+ requests = []
57
+ records.each do |record|
58
+ request = {
59
+ image: {},
60
+ features: @features
61
+ }
62
+ image_body = get_image_body(record)
63
+ request[:image][:content] = Base64.encode64(image_body)
64
+ requests << request
65
+ end
66
+ body = {
67
+ requests: requests
68
+ }
69
+ @post.body = body.to_json
70
+ Embulk.logger.debug "request body => #{@post.body}"
71
+
72
+ response_hash = {}
73
+ @http.start do |h|
74
+ response = h.request(@post)
75
+ response_hash = JSON.parse(response.body)
76
+ end
77
+ records.each_with_index do |record, i|
78
+ recognized = response_hash['responses'][i]
79
+ Embulk.logger.warn "Error image => [#{record[@image_path_key_name]}] #{recognized}" if recognized.key?("error")
80
+ page_builder.add(record.values + [recognized])
81
+ end
82
+
83
+ sleep @delay
84
+ end
85
+ end
86
+
87
+ def finish
88
+ page_builder.finish
89
+ end
90
+
91
+ private
92
+ def get_image_body(record)
93
+ image_path = record[@image_path_key_name]
94
+ if image_path =~ /https?\:\/\//
95
+ Net::HTTP.get_response(URI.parse(image_path)).body rescue ""
96
+ else
97
+ File.read(image_path) rescue ""
98
+ end
99
+ end
100
+ end
101
+ end
102
+ end
metadata ADDED
@@ -0,0 +1,108 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: embulk-filter-google_vision_api
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - toyama0919
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2017-01-16 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - ">="
17
+ - !ruby/object:Gem::Version
18
+ version: '0'
19
+ name: jruby-openssl
20
+ prerelease: false
21
+ type: :runtime
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ requirement: !ruby/object:Gem::Requirement
29
+ requirements:
30
+ - - ">="
31
+ - !ruby/object:Gem::Version
32
+ version: 0.8.15
33
+ name: embulk
34
+ prerelease: false
35
+ type: :development
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: 0.8.15
41
+ - !ruby/object:Gem::Dependency
42
+ requirement: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - ">="
45
+ - !ruby/object:Gem::Version
46
+ version: 1.10.6
47
+ name: bundler
48
+ prerelease: false
49
+ type: :development
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: 1.10.6
55
+ - !ruby/object:Gem::Dependency
56
+ requirement: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - ">="
59
+ - !ruby/object:Gem::Version
60
+ version: '10.0'
61
+ name: rake
62
+ prerelease: false
63
+ type: :development
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '10.0'
69
+ description: Google Vision Api filter plugin for Embulk. Very easy image recognition.
70
+ email:
71
+ - toyama0919@gmail.com
72
+ executables: []
73
+ extensions: []
74
+ extra_rdoc_files: []
75
+ files:
76
+ - ".gitignore"
77
+ - ".ruby-version"
78
+ - Gemfile
79
+ - LICENSE.txt
80
+ - README.md
81
+ - Rakefile
82
+ - embulk-filter-google_vision_api.gemspec
83
+ - lib/embulk/filter/google_vision_api.rb
84
+ homepage: https://github.com/toyama0919/embulk-filter-google_vision_api
85
+ licenses:
86
+ - MIT
87
+ metadata: {}
88
+ post_install_message:
89
+ rdoc_options: []
90
+ require_paths:
91
+ - lib
92
+ required_ruby_version: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ required_rubygems_version: !ruby/object:Gem::Requirement
98
+ requirements:
99
+ - - ">="
100
+ - !ruby/object:Gem::Version
101
+ version: '0'
102
+ requirements: []
103
+ rubyforge_project:
104
+ rubygems_version: 2.6.6
105
+ signing_key:
106
+ specification_version: 4
107
+ summary: Google Vision Api filter plugin for Embulk
108
+ test_files: []