embulk-input-gmail 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +13 -0
- data/LICENSE.txt +21 -0
- data/README.md +86 -0
- data/build.gradle +103 -0
- data/config/checkstyle/checkstyle.xml +128 -0
- data/config/checkstyle/default.xml +108 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +5 -0
- data/gradlew +172 -0
- data/gradlew.bat +84 -0
- data/lib/embulk/input/gmail.rb +3 -0
- data/src/main/java/org/embulk/input/gmail/GmailInputPlugin.java +280 -0
- data/src/main/java/org/embulk/input/gmail/GmailWrapper.java +142 -0
- data/src/main/java/org/embulk/input/gmail/GoogleCredentialCreator.java +56 -0
- data/src/test/java/org/embulk/input/gmail/TestGmailInputPlugin.java +5 -0
- data/src/test/java/org/embulk/input/gmail/TestGoogleCredentialCreator.java +23 -0
- data/src/test/resources/.gitignore +3 -0
- metadata +105 -0
@@ -0,0 +1,142 @@
|
|
1
|
+
package org.embulk.input.gmail;
|
2
|
+
|
3
|
+
import java.io.IOException;
|
4
|
+
import java.nio.file.Paths;
|
5
|
+
import java.security.GeneralSecurityException;
|
6
|
+
import java.util.ArrayList;
|
7
|
+
import java.util.Collections;
|
8
|
+
import java.util.HashMap;
|
9
|
+
import java.util.List;
|
10
|
+
import java.util.Map;
|
11
|
+
import java.util.Optional;
|
12
|
+
|
13
|
+
import com.google.api.client.auth.oauth2.Credential;
|
14
|
+
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
|
15
|
+
import com.google.api.client.http.javanet.NetHttpTransport;
|
16
|
+
import com.google.api.client.json.JsonFactory;
|
17
|
+
import com.google.api.client.json.jackson2.JacksonFactory;
|
18
|
+
import com.google.api.services.gmail.Gmail;
|
19
|
+
import com.google.api.services.gmail.model.ListMessagesResponse;
|
20
|
+
import com.google.api.services.gmail.model.Message;
|
21
|
+
import com.google.api.services.gmail.model.MessagePart;
|
22
|
+
import com.google.api.services.gmail.model.MessagePartHeader;
|
23
|
+
|
24
|
+
/**
|
25
|
+
* GmailWrapper
|
26
|
+
*/
|
27
|
+
public class GmailWrapper {
|
28
|
+
private static final String APPLICATION_NAME = "Embulk Input Plugin Gmail";
|
29
|
+
|
30
|
+
private final Gmail service;
|
31
|
+
|
32
|
+
/**
|
33
|
+
* Constructor
|
34
|
+
*/
|
35
|
+
public GmailWrapper(String clientSecretPath, String tokensDirectory) throws IOException, GeneralSecurityException {
|
36
|
+
Credential credential = GoogleCredentialCreator.getCredentials(Paths.get(clientSecretPath), tokensDirectory);
|
37
|
+
NetHttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
|
38
|
+
JsonFactory jsonFactory = JacksonFactory.getDefaultInstance();
|
39
|
+
|
40
|
+
this.service = new Gmail.Builder(transport, jsonFactory, credential)
|
41
|
+
.setApplicationName(APPLICATION_NAME).build();
|
42
|
+
}
|
43
|
+
|
44
|
+
public Result search(String user, String query) throws IOException {
|
45
|
+
ListMessagesResponse listResponse = service.users().messages().list(user).setQ(query).execute();
|
46
|
+
List<com.google.api.services.gmail.model.Message> messages = listResponse.getMessages();
|
47
|
+
|
48
|
+
if(messages == null || messages.isEmpty()) {
|
49
|
+
return new Result(
|
50
|
+
Collections.emptyList(),
|
51
|
+
Collections.emptyList());
|
52
|
+
}
|
53
|
+
|
54
|
+
List<GmailWrapper.Message> success = new ArrayList<>();
|
55
|
+
List<GmailWrapper.Message> failed = new ArrayList<>();
|
56
|
+
for (com.google.api.services.gmail.model.Message message : messages) {
|
57
|
+
// get full message.
|
58
|
+
com.google.api.services.gmail.model.Message fullMessage = service.users().messages().get(user, message.getId()).execute();
|
59
|
+
|
60
|
+
// get Payload.
|
61
|
+
MessagePart payload = fullMessage.getPayload();
|
62
|
+
|
63
|
+
// put Headers.
|
64
|
+
Map<String, String> headers = new HashMap<>();
|
65
|
+
for (MessagePartHeader header : payload.getHeaders()) {
|
66
|
+
headers.put(header.getName(), header.getValue());
|
67
|
+
}
|
68
|
+
|
69
|
+
// get Body of `text/plain`.
|
70
|
+
// supported structure:
|
71
|
+
// 1. text/plain
|
72
|
+
// 2. multipart/alternative
|
73
|
+
// - text/plain
|
74
|
+
// 3. multipart/mixed
|
75
|
+
// - text/plain
|
76
|
+
MessagePart part;
|
77
|
+
if (payload.getParts() == null) {
|
78
|
+
part = payload;
|
79
|
+
if (!(part.getMimeType().equals("text/plain"))) {
|
80
|
+
failed.add(new GmailWrapper.Message(headers, null));
|
81
|
+
continue;
|
82
|
+
}
|
83
|
+
} else {
|
84
|
+
Optional<MessagePart> bodyPart = payload.getParts().stream().filter(p -> p.getMimeType().equals("text/plain")).findFirst();
|
85
|
+
if (!(bodyPart.isPresent())) {
|
86
|
+
failed.add(new GmailWrapper.Message(headers, null));
|
87
|
+
continue;
|
88
|
+
}
|
89
|
+
part = bodyPart.get();
|
90
|
+
}
|
91
|
+
|
92
|
+
// get body string
|
93
|
+
byte[] rawBody = part.getBody().decodeData();
|
94
|
+
String body = new String(rawBody, "UTF-8");
|
95
|
+
|
96
|
+
success.add(new Message(headers, body));
|
97
|
+
}
|
98
|
+
|
99
|
+
Result result = new Result(success, failed);
|
100
|
+
return result;
|
101
|
+
}
|
102
|
+
|
103
|
+
class Result {
|
104
|
+
private List<GmailWrapper.Message> success;
|
105
|
+
private List<GmailWrapper.Message> failed;
|
106
|
+
|
107
|
+
public Result(
|
108
|
+
List<GmailWrapper.Message> success,
|
109
|
+
List<GmailWrapper.Message> failed) {
|
110
|
+
this.success = success;
|
111
|
+
this.failed = failed;
|
112
|
+
}
|
113
|
+
|
114
|
+
public List<GmailWrapper.Message> getSuccessMessages() {
|
115
|
+
return success;
|
116
|
+
}
|
117
|
+
|
118
|
+
public List<GmailWrapper.Message> getFailedMessages() {
|
119
|
+
return failed;
|
120
|
+
}
|
121
|
+
}
|
122
|
+
|
123
|
+
class Message {
|
124
|
+
private Map<String, String> headers;
|
125
|
+
private Optional<String> body;
|
126
|
+
|
127
|
+
public Message(
|
128
|
+
Map<String, String> headers,
|
129
|
+
String body) {
|
130
|
+
this.headers = Optional.of(headers).get();
|
131
|
+
this.body = Optional.ofNullable(body);
|
132
|
+
}
|
133
|
+
|
134
|
+
public Map<String, String> getHeaders() {
|
135
|
+
return headers;
|
136
|
+
}
|
137
|
+
|
138
|
+
public Optional<String> getBody() {
|
139
|
+
return body;
|
140
|
+
}
|
141
|
+
}
|
142
|
+
}
|
@@ -0,0 +1,56 @@
|
|
1
|
+
package org.embulk.input.gmail;
|
2
|
+
|
3
|
+
import java.io.BufferedReader;
|
4
|
+
import java.io.File;
|
5
|
+
import java.io.IOException;
|
6
|
+
import java.nio.file.Files;
|
7
|
+
import java.nio.file.Path;
|
8
|
+
import java.nio.file.Paths;
|
9
|
+
import java.security.GeneralSecurityException;
|
10
|
+
import java.util.Collections;
|
11
|
+
import java.util.List;
|
12
|
+
|
13
|
+
import com.google.api.client.auth.oauth2.Credential;
|
14
|
+
import com.google.api.client.extensions.java6.auth.oauth2.AuthorizationCodeInstalledApp;
|
15
|
+
import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow;
|
16
|
+
import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets;
|
17
|
+
import com.google.api.client.googleapis.extensions.java6.auth.oauth2.GooglePromptReceiver;
|
18
|
+
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
|
19
|
+
import com.google.api.client.http.javanet.NetHttpTransport;
|
20
|
+
import com.google.api.client.json.JsonFactory;
|
21
|
+
import com.google.api.client.json.jackson2.JacksonFactory;
|
22
|
+
import com.google.api.client.util.store.FileDataStoreFactory;
|
23
|
+
import com.google.api.services.gmail.GmailScopes;
|
24
|
+
|
25
|
+
/**
|
26
|
+
* GoogleCredentialCreator
|
27
|
+
*/
|
28
|
+
public class GoogleCredentialCreator {
|
29
|
+
|
30
|
+
private static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance();
|
31
|
+
private static final List<String> SCOPES = Collections.singletonList(GmailScopes.GMAIL_READONLY);
|
32
|
+
|
33
|
+
private GoogleCredentialCreator() {}
|
34
|
+
|
35
|
+
public static Credential getCredentials(Path clientSecretPath, String tokenDirectory) throws IOException, GeneralSecurityException {
|
36
|
+
NetHttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
|
37
|
+
GoogleClientSecrets clientSecret;
|
38
|
+
try (BufferedReader br = Files.newBufferedReader(clientSecretPath)) {
|
39
|
+
clientSecret = GoogleClientSecrets.load(JSON_FACTORY, br);
|
40
|
+
|
41
|
+
GoogleAuthorizationCodeFlow flow = new GoogleAuthorizationCodeFlow.Builder(
|
42
|
+
transport, JSON_FACTORY, clientSecret, SCOPES)
|
43
|
+
.setDataStoreFactory(new FileDataStoreFactory(new File(tokenDirectory)))
|
44
|
+
.setAccessType("offline")
|
45
|
+
.build();
|
46
|
+
GooglePromptReceiver receier = new GooglePromptReceiver();
|
47
|
+
return new AuthorizationCodeInstalledApp(flow, receier).authorize("user");
|
48
|
+
}
|
49
|
+
}
|
50
|
+
|
51
|
+
public static void main(String[] args) throws IOException, GeneralSecurityException {
|
52
|
+
GoogleCredentialCreator.getCredentials(
|
53
|
+
Paths.get(args[0]),
|
54
|
+
args[1]);
|
55
|
+
}
|
56
|
+
}
|
@@ -0,0 +1,23 @@
|
|
1
|
+
package org.embulk.input.gmail;
|
2
|
+
|
3
|
+
import java.io.IOException;
|
4
|
+
import java.nio.file.Paths;
|
5
|
+
import java.security.GeneralSecurityException;
|
6
|
+
|
7
|
+
import static org.junit.Assert.*;
|
8
|
+
|
9
|
+
import org.junit.Test;
|
10
|
+
|
11
|
+
/**
|
12
|
+
* TestGoogleCredentialCreator
|
13
|
+
*/
|
14
|
+
public class TestGoogleCredentialCreator {
|
15
|
+
@Test
|
16
|
+
public void testGetCredentials() {
|
17
|
+
try {
|
18
|
+
GoogleCredentialCreator.getCredentials(Paths.get("./src/test/resources/client_secret.json"), "./src/test/resources/tokens");
|
19
|
+
} catch (IOException|GeneralSecurityException e) {
|
20
|
+
// TODO: fail
|
21
|
+
}
|
22
|
+
}
|
23
|
+
}
|
metadata
ADDED
@@ -0,0 +1,105 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: embulk-input-gmail
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- mikoto2000
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2018-11-02 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
15
|
+
requirements:
|
16
|
+
- - "~>"
|
17
|
+
- !ruby/object:Gem::Version
|
18
|
+
version: '1.0'
|
19
|
+
name: bundler
|
20
|
+
prerelease: false
|
21
|
+
type: :development
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
requirement: !ruby/object:Gem::Requirement
|
29
|
+
requirements:
|
30
|
+
- - ">="
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '10.0'
|
33
|
+
name: rake
|
34
|
+
prerelease: false
|
35
|
+
type: :development
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
|
+
description: Loads records from Gmail.
|
42
|
+
email:
|
43
|
+
- mikoto2000@gmail.com
|
44
|
+
executables: []
|
45
|
+
extensions: []
|
46
|
+
extra_rdoc_files: []
|
47
|
+
files:
|
48
|
+
- ".gitignore"
|
49
|
+
- LICENSE.txt
|
50
|
+
- README.md
|
51
|
+
- build.gradle
|
52
|
+
- classpath/commons-codec-1.10.jar
|
53
|
+
- classpath/commons-logging-1.2.jar
|
54
|
+
- classpath/embulk-input-gmail-0.1.0.jar
|
55
|
+
- classpath/google-api-client-1.26.0.jar
|
56
|
+
- classpath/google-api-client-java6-1.26.0.jar
|
57
|
+
- classpath/google-api-services-gmail-v1-rev83-1.23.0.jar
|
58
|
+
- classpath/google-http-client-1.26.0.jar
|
59
|
+
- classpath/google-http-client-jackson2-1.26.0.jar
|
60
|
+
- classpath/google-oauth-client-1.26.0.jar
|
61
|
+
- classpath/google-oauth-client-java6-1.26.0.jar
|
62
|
+
- classpath/guava-20.0.jar
|
63
|
+
- classpath/httpclient-4.5.5.jar
|
64
|
+
- classpath/httpcore-4.4.9.jar
|
65
|
+
- classpath/j2objc-annotations-1.1.jar
|
66
|
+
- classpath/jackson-core-2.9.6.jar
|
67
|
+
- classpath/jsr305-3.0.2.jar
|
68
|
+
- config/checkstyle/checkstyle.xml
|
69
|
+
- config/checkstyle/default.xml
|
70
|
+
- gradle/wrapper/gradle-wrapper.jar
|
71
|
+
- gradle/wrapper/gradle-wrapper.properties
|
72
|
+
- gradlew
|
73
|
+
- gradlew.bat
|
74
|
+
- lib/embulk/input/gmail.rb
|
75
|
+
- src/main/java/org/embulk/input/gmail/GmailInputPlugin.java
|
76
|
+
- src/main/java/org/embulk/input/gmail/GmailWrapper.java
|
77
|
+
- src/main/java/org/embulk/input/gmail/GoogleCredentialCreator.java
|
78
|
+
- src/test/java/org/embulk/input/gmail/TestGmailInputPlugin.java
|
79
|
+
- src/test/java/org/embulk/input/gmail/TestGoogleCredentialCreator.java
|
80
|
+
- src/test/resources/.gitignore
|
81
|
+
homepage: https://github.com/mikoto2000/embulk-input-gmail
|
82
|
+
licenses:
|
83
|
+
- MIT
|
84
|
+
metadata: {}
|
85
|
+
post_install_message:
|
86
|
+
rdoc_options: []
|
87
|
+
require_paths:
|
88
|
+
- lib
|
89
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
90
|
+
requirements:
|
91
|
+
- - ">="
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: '0'
|
94
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
95
|
+
requirements:
|
96
|
+
- - ">="
|
97
|
+
- !ruby/object:Gem::Version
|
98
|
+
version: '0'
|
99
|
+
requirements: []
|
100
|
+
rubyforge_project:
|
101
|
+
rubygems_version: 2.6.8
|
102
|
+
signing_key:
|
103
|
+
specification_version: 4
|
104
|
+
summary: Gmail input plugin for Embulk
|
105
|
+
test_files: []
|