embulk-input-gmail 0.1.0

data/src/main/java/org/embulk/input/gmail/GmailWrapper.java

@@ -0,0 +1,142 @@
+package org.embulk.input.gmail;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import com.google.api.client.auth.oauth2.Credential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.gmail.Gmail;
+import com.google.api.services.gmail.model.ListMessagesResponse;
+import com.google.api.services.gmail.model.Message;
+import com.google.api.services.gmail.model.MessagePart;
+import com.google.api.services.gmail.model.MessagePartHeader;
+
+/**
+ * GmailWrapper
+ */
+public class GmailWrapper {
+    private static final String APPLICATION_NAME = "Embulk Input Plugin Gmail";
+
+    private final Gmail service;
+
+    /**
+     * Constructor
+     */
+    public GmailWrapper(String clientSecretPath, String tokensDirectory) throws IOException, GeneralSecurityException {
+        Credential credential = GoogleCredentialCreator.getCredentials(Paths.get(clientSecretPath), tokensDirectory);
+        NetHttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
+        JsonFactory jsonFactory = JacksonFactory.getDefaultInstance();
+
+        this.service = new Gmail.Builder(transport, jsonFactory, credential)
+                .setApplicationName(APPLICATION_NAME).build();
+    }
+
+    public Result search(String user, String query) throws IOException {
+        ListMessagesResponse listResponse = service.users().messages().list(user).setQ(query).execute();
+        List<com.google.api.services.gmail.model.Message> messages = listResponse.getMessages();
+
+        if(messages == null || messages.isEmpty()) {
+            return new Result(
+                    Collections.emptyList(),
+                    Collections.emptyList());
+        }
+
+        List<GmailWrapper.Message> success = new ArrayList<>();
+        List<GmailWrapper.Message> failed = new ArrayList<>();
+        for (com.google.api.services.gmail.model.Message message : messages) {
+            // get full message.
+            com.google.api.services.gmail.model.Message fullMessage = service.users().messages().get(user, message.getId()).execute();
+
+            // get Payload.
+            MessagePart payload = fullMessage.getPayload();
+
+            // put Headers.
+            Map<String, String> headers = new HashMap<>();
+            for (MessagePartHeader header : payload.getHeaders()) {
+                headers.put(header.getName(), header.getValue());
+            }
+
+            // get Body of `text/plain`.
+            // supported structure:
+            //     1. text/plain
+            //     2. multipart/alternative
+            //         - text/plain
+            //     3. multipart/mixed
+            //         - text/plain
+            MessagePart part;
+            if (payload.getParts() == null) {
+                part = payload;
+                if (!(part.getMimeType().equals("text/plain"))) {
+                    failed.add(new GmailWrapper.Message(headers, null));
+                    continue;
+                }
+            } else {
+                Optional<MessagePart> bodyPart = payload.getParts().stream().filter(p -> p.getMimeType().equals("text/plain")).findFirst();
+                if (!(bodyPart.isPresent())) {
+                    failed.add(new GmailWrapper.Message(headers, null));
+                    continue;
+                }
+                part = bodyPart.get();
+            }
+
+            // get body string
+            byte[] rawBody = part.getBody().decodeData();
+            String body = new String(rawBody, "UTF-8");
+
+            success.add(new Message(headers, body));
+        }
+
+        Result result = new Result(success, failed);
+        return result;
+    }
+
+    class Result {
+        private List<GmailWrapper.Message> success;
+        private List<GmailWrapper.Message> failed;
+
+        public Result(
+                List<GmailWrapper.Message> success,
+                List<GmailWrapper.Message> failed) {
+            this.success = success;
+            this.failed = failed;
+        }
+
+        public List<GmailWrapper.Message> getSuccessMessages() {
+            return success;
+        }
+
+        public List<GmailWrapper.Message> getFailedMessages() {
+            return failed;
+        }
+    }
+
+    class Message {
+        private Map<String, String> headers;
+        private Optional<String> body;
+
+        public Message(
+                Map<String, String> headers,
+                String body) {
+            this.headers = Optional.of(headers).get();
+            this.body = Optional.ofNullable(body);
+        }
+
+        public Map<String, String> getHeaders() {
+            return headers;
+        }
+
+        public Optional<String> getBody() {
+            return body;
+        }
+    }
+}

data/src/main/java/org/embulk/input/gmail/GoogleCredentialCreator.java

@@ -0,0 +1,56 @@
+package org.embulk.input.gmail;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.GeneralSecurityException;
+import java.util.Collections;
+import java.util.List;
+
+import com.google.api.client.auth.oauth2.Credential;
+import com.google.api.client.extensions.java6.auth.oauth2.AuthorizationCodeInstalledApp;
+import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow;
+import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets;
+import com.google.api.client.googleapis.extensions.java6.auth.oauth2.GooglePromptReceiver;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.client.util.store.FileDataStoreFactory;
+import com.google.api.services.gmail.GmailScopes;
+
+/**
+ * GoogleCredentialCreator
+ */
+public class GoogleCredentialCreator {
+
+    private static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance();
+    private static final List<String> SCOPES = Collections.singletonList(GmailScopes.GMAIL_READONLY);
+
+    private GoogleCredentialCreator() {}
+
+    public static Credential getCredentials(Path clientSecretPath, String tokenDirectory) throws IOException, GeneralSecurityException {
+        NetHttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
+        GoogleClientSecrets clientSecret;
+        try (BufferedReader br = Files.newBufferedReader(clientSecretPath)) {
+            clientSecret = GoogleClientSecrets.load(JSON_FACTORY, br);
+
+            GoogleAuthorizationCodeFlow flow = new GoogleAuthorizationCodeFlow.Builder(
+                    transport, JSON_FACTORY, clientSecret, SCOPES)
+                    .setDataStoreFactory(new FileDataStoreFactory(new File(tokenDirectory)))
+                    .setAccessType("offline")
+                    .build();
+            GooglePromptReceiver receier = new GooglePromptReceiver();
+            return new AuthorizationCodeInstalledApp(flow, receier).authorize("user");
+        }
+    }
+
+    public static void main(String[] args) throws IOException, GeneralSecurityException {
+        GoogleCredentialCreator.getCredentials(
+                Paths.get(args[0]),
+                args[1]);
+    }
+}

data/src/test/java/org/embulk/input/gmail/TestGmailInputPlugin.java

@@ -0,0 +1,5 @@
+package org.embulk.input.gmail;
+
+public class TestGmailInputPlugin
+{
+}

data/src/test/java/org/embulk/input/gmail/TestGoogleCredentialCreator.java

@@ -0,0 +1,23 @@
+package org.embulk.input.gmail;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.security.GeneralSecurityException;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+/**
+ * TestGoogleCredentialCreator
+ */
+public class TestGoogleCredentialCreator {
+    @Test
+    public void testGetCredentials() {
+        try {
+            GoogleCredentialCreator.getCredentials(Paths.get("./src/test/resources/client_secret.json"), "./src/test/resources/tokens");
+        } catch (IOException|GeneralSecurityException e) {
+            // TODO: fail
+        }
+    }
+}

data/src/test/resources/.gitignore

@@ -0,0 +1,3 @@
+client_secret.json
+tokens/
+

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification
+name: embulk-input-gmail
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- mikoto2000
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2018-11-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Loads records from Gmail.
+email:
+- mikoto2000@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- LICENSE.txt
+- README.md
+- build.gradle
+- classpath/commons-codec-1.10.jar
+- classpath/commons-logging-1.2.jar
+- classpath/embulk-input-gmail-0.1.0.jar
+- classpath/google-api-client-1.26.0.jar
+- classpath/google-api-client-java6-1.26.0.jar
+- classpath/google-api-services-gmail-v1-rev83-1.23.0.jar
+- classpath/google-http-client-1.26.0.jar
+- classpath/google-http-client-jackson2-1.26.0.jar
+- classpath/google-oauth-client-1.26.0.jar
+- classpath/google-oauth-client-java6-1.26.0.jar
+- classpath/guava-20.0.jar
+- classpath/httpclient-4.5.5.jar
+- classpath/httpcore-4.4.9.jar
+- classpath/j2objc-annotations-1.1.jar
+- classpath/jackson-core-2.9.6.jar
+- classpath/jsr305-3.0.2.jar
+- config/checkstyle/checkstyle.xml
+- config/checkstyle/default.xml
+- gradle/wrapper/gradle-wrapper.jar
+- gradle/wrapper/gradle-wrapper.properties
+- gradlew
+- gradlew.bat
+- lib/embulk/input/gmail.rb
+- src/main/java/org/embulk/input/gmail/GmailInputPlugin.java
+- src/main/java/org/embulk/input/gmail/GmailWrapper.java
+- src/main/java/org/embulk/input/gmail/GoogleCredentialCreator.java
+- src/test/java/org/embulk/input/gmail/TestGmailInputPlugin.java
+- src/test/java/org/embulk/input/gmail/TestGoogleCredentialCreator.java
+- src/test/resources/.gitignore
+homepage: https://github.com/mikoto2000/embulk-input-gmail
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.8
+signing_key:
+specification_version: 4
+summary: Gmail input plugin for Embulk
+test_files: []