embulk-input-firestore 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (83) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +2 -0
  3. data/.scalafmt.conf +13 -0
  4. data/LICENSE +21 -0
  5. data/README.md +2 -0
  6. data/build.gradle +91 -0
  7. data/classpath/animal-sniffer-annotations-1.17.jar +0 -0
  8. data/classpath/annotations-4.1.1.4.jar +0 -0
  9. data/classpath/api-common-1.8.1.jar +0 -0
  10. data/classpath/auto-value-annotations-1.6.6.jar +0 -0
  11. data/classpath/checker-qual-2.8.1.jar +0 -0
  12. data/classpath/commons-codec-1.11.jar +0 -0
  13. data/classpath/commons-lang3-3.5.jar +0 -0
  14. data/classpath/commons-logging-1.2.jar +0 -0
  15. data/classpath/commons-math3-3.6.1.jar +0 -0
  16. data/classpath/embulk-input-firestore-0.0.1-shadow.jar +0 -0
  17. data/classpath/error_prone_annotations-2.3.2.jar +0 -0
  18. data/classpath/failureaccess-1.0.1.jar +0 -0
  19. data/classpath/gax-1.48.1.jar +0 -0
  20. data/classpath/gax-grpc-1.48.1.jar +0 -0
  21. data/classpath/gax-httpjson-0.65.1.jar +0 -0
  22. data/classpath/google-api-client-1.30.2.jar +0 -0
  23. data/classpath/google-api-client-gson-1.30.1.jar +0 -0
  24. data/classpath/google-api-services-storage-v1-rev20190624-1.30.1.jar +0 -0
  25. data/classpath/google-auth-library-credentials-0.17.1.jar +0 -0
  26. data/classpath/google-auth-library-oauth2-http-0.17.1.jar +0 -0
  27. data/classpath/google-cloud-core-1.90.0.jar +0 -0
  28. data/classpath/google-cloud-core-grpc-1.90.0.jar +0 -0
  29. data/classpath/google-cloud-core-http-1.90.0.jar +0 -0
  30. data/classpath/google-cloud-storage-1.91.0.jar +0 -0
  31. data/classpath/google-http-client-1.31.0.jar +0 -0
  32. data/classpath/google-http-client-appengine-1.31.0.jar +0 -0
  33. data/classpath/google-http-client-gson-1.30.1.jar +0 -0
  34. data/classpath/google-http-client-jackson2-1.31.0.jar +0 -0
  35. data/classpath/google-oauth-client-1.30.1.jar +0 -0
  36. data/classpath/grpc-alts-1.23.0.jar +0 -0
  37. data/classpath/grpc-auth-1.23.0.jar +0 -0
  38. data/classpath/grpc-context-1.23.0.jar +0 -0
  39. data/classpath/grpc-grpclb-1.23.0.jar +0 -0
  40. data/classpath/grpc-protobuf-1.23.0.jar +0 -0
  41. data/classpath/grpc-protobuf-lite-1.23.0.jar +0 -0
  42. data/classpath/grpc-stub-1.23.0.jar +0 -0
  43. data/classpath/gson-2.8.5.jar +0 -0
  44. data/classpath/httpclient-4.5.9.jar +0 -0
  45. data/classpath/httpcore-4.4.11.jar +0 -0
  46. data/classpath/j2objc-annotations-1.3.jar +0 -0
  47. data/classpath/jackson-core-2.9.9.jar +0 -0
  48. data/classpath/javax.annotation-api-1.3.2.jar +0 -0
  49. data/classpath/jsqlparser-3.1.jar +0 -0
  50. data/classpath/jsr305-3.0.2.jar +0 -0
  51. data/classpath/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar +0 -0
  52. data/classpath/netty-buffer-4.1.34.Final.jar +0 -0
  53. data/classpath/netty-codec-4.1.34.Final.jar +0 -0
  54. data/classpath/netty-codec-http-4.1.34.Final.jar +0 -0
  55. data/classpath/netty-common-4.1.34.Final.jar +0 -0
  56. data/classpath/netty-handler-4.1.34.Final.jar +0 -0
  57. data/classpath/netty-resolver-4.1.34.Final.jar +0 -0
  58. data/classpath/netty-transport-4.1.34.Final.jar +0 -0
  59. data/classpath/opencensus-api-0.23.0.jar +0 -0
  60. data/classpath/opencensus-contrib-grpc-metrics-0.21.0.jar +0 -0
  61. data/classpath/opencensus-contrib-grpc-util-0.23.0.jar +0 -0
  62. data/classpath/opencensus-contrib-http-util-0.21.0.jar +0 -0
  63. data/classpath/perfmark-api-0.17.0.jar +0 -0
  64. data/classpath/proto-google-cloud-firestore-admin-v1-1.21.0.jar +0 -0
  65. data/classpath/proto-google-cloud-firestore-v1-1.21.0.jar +0 -0
  66. data/classpath/proto-google-cloud-firestore-v1beta1-0.74.0.jar +0 -0
  67. data/classpath/proto-google-common-protos-1.16.0.jar +0 -0
  68. data/classpath/proto-google-iam-v1-0.12.0.jar +0 -0
  69. data/classpath/protobuf-java-3.9.1.jar +0 -0
  70. data/classpath/protobuf-java-util-3.9.1.jar +0 -0
  71. data/classpath/scala-library-2.12.3.jar +0 -0
  72. data/classpath/slf4j-api-1.7.25.jar +0 -0
  73. data/classpath/threetenbp-1.3.3.jar +0 -0
  74. data/examples/firestore2stdout.yaml +8 -0
  75. data/gradle.properties +1 -0
  76. data/gradle/wrapper/gradle-wrapper.jar +0 -0
  77. data/gradle/wrapper/gradle-wrapper.properties +6 -0
  78. data/gradlew +172 -0
  79. data/gradlew.bat +84 -0
  80. data/lib/embulk/input/firestore.rb +3 -0
  81. data/src/main/scala/org/embulk/input/firestore/FirestoreInputPlugin.scala +102 -0
  82. data/src/main/scala/org/embulk/input/firestore/PluginTask.scala +23 -0
  83. metadata +125 -0
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: fee292d3a8ec98d9fd54b54f164a4f6d3a081266f60317fc928d51277a48c7a2
4
+ data.tar.gz: fc811f4a0d4f84210d474b3d576aa7e1553b818de3298d86dbb3ec7b365844a8
5
+ SHA512:
6
+ metadata.gz: d66db9159e3609029ca3eabb1fc79a3ed85cd936a1ca38f41d33e7fdec5ce5a14b301bdc2fa3d465cbee939f3cab3cde9ac86d5432252823d56daad5c088966d
7
+ data.tar.gz: 02d238fe94eb849fc1639b3312509ba6ebd1a239aea22f9d939471daadddb9cdfd95fe596656f88513c47d02590a22ac9b8142f0426895f46dd7cd861ac81f32
@@ -0,0 +1,2 @@
1
+ *.class
2
+ *.log
@@ -0,0 +1,13 @@
1
+ version=2.3.2
2
+ project.git = true
3
+ project.excludeFilters = [
4
+ scalafmt-benchmarks/src/resources,
5
+ sbt-test
6
+ bin/issue
7
+ ]
8
+ align = none
9
+ # Disabled in default since this operation is potentially
10
+ # dangerous if you define your own stripMargin with different
11
+ # semantics from the stdlib stripMargin.
12
+ assumeStandardLibraryStripMargin = true
13
+ onTestFailure = "To fix this, run ./scalafmt from the project root directory"
data/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2019 Ryo Okubo
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,2 @@
1
+ # embulk-input-firestore
2
+ [WIP] Cloud Firestore input plugin for Embulk loads records from Google Cloud Firestore.
@@ -0,0 +1,91 @@
1
+ plugins {
2
+ id 'scala'
3
+ id 'com.github.jruby-gradle.base' version '1.6.0'
4
+ id 'com.github.johnrengelman.shadow' version '5.2.0'
5
+ id 'cz.alenkacz.gradle.scalafmt' version '1.10.0'
6
+ }
7
+ import com.github.jrubygradle.JRubyExec
8
+
9
+ configurations {
10
+ provided
11
+ }
12
+
13
+ repositories {
14
+ mavenCentral()
15
+ mavenLocal()
16
+ jcenter()
17
+ }
18
+
19
+ // Relocate Guava packages since it's incompatible with Guava's version from Embulk
20
+ shadowJar {
21
+ classifier 'shadow'
22
+
23
+ dependencies {
24
+ include dependency('com.google.guava:guava')
25
+
26
+ include dependency('com.google.cloud:google-cloud-firestore')
27
+ include dependency('com.google.firebase:firebase-admin')
28
+ include dependency('com.syucream:firesql')
29
+
30
+ include dependency('io.grpc:grpc-netty-shaded')
31
+ include dependency('io.grpc:grpc-core')
32
+ include dependency('io.grpc:grpc-api')
33
+ }
34
+
35
+ relocate 'com.google.common', 'relocated.com.google.common'
36
+ }
37
+
38
+ sourceCompatibility = 1.8
39
+ targetCompatibility = 1.8
40
+
41
+ dependencies {
42
+ compile 'org.scala-lang:scala-library:2.12.3'
43
+
44
+ compile 'org.embulk:embulk-core:0.9.12'
45
+ provided 'org.embulk:embulk-core:0.9.12'
46
+
47
+ // Should be shadowed
48
+ compile 'com.google.firebase:firebase-admin:6.11.0'
49
+ compile 'com.syucream:firesql:0.0.1'
50
+ }
51
+
52
+ task classpath(type: Copy, dependsOn: ['jar', 'shadowJar']) {
53
+ doFirst { file('classpath').deleteDir() }
54
+
55
+ from (configurations.runtime
56
+ - configurations.provided
57
+ + configurations.shadow
58
+ - files(shadowJar.getIncludedDependencies())
59
+ + files(shadowJar.archiveFile))
60
+
61
+ into 'classpath'
62
+ }
63
+ clean { delete 'classpath' }
64
+
65
+ task gem(type: JRubyExec, dependsOn: ['build', 'gemspec', 'classpath']) {
66
+ script 'gem'
67
+ scriptArgs 'build', 'build/gemspec'
68
+ doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: 'pkg') }
69
+ }
70
+
71
+ task gemspec {
72
+ doLast {
73
+ file('build').mkdirs()
74
+ file('build/gemspec').write($/
75
+ Gem::Specification.new do |spec|
76
+ spec.name = "${project.name}"
77
+ spec.version = "${project.version}"
78
+ spec.authors = ["Ryo Okubo"]
79
+ spec.summary = %[Cloud Firestoreinput plugin for Embulk]
80
+ spec.description = %[Selects records from a table.]
81
+ spec.email = ["syucream@gmail.com"]
82
+ spec.licenses = ["Apache-2.0"]
83
+ spec.homepage = "https://github.com/syucream/embulk-input-firestore"
84
+
85
+ spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
86
+ spec.test_files = spec.files.grep(%r"^(test|spec)/")
87
+ spec.require_paths = ["lib"]
88
+ end
89
+ /$)
90
+ }
91
+ }
Binary file
Binary file
@@ -0,0 +1,8 @@
1
+ in:
2
+ type: firestore
3
+ project_id: "syucream-firebase-dev"
4
+ json_keyfile: credential.json
5
+ sql: "SELECT id, name FROM users"
6
+
7
+ out:
8
+ type: stdout
@@ -0,0 +1 @@
1
+ version=0.0.1
@@ -0,0 +1,6 @@
1
+ #Mon Oct 21 20:40:52 JST 2019
2
+ distributionUrl=https\://services.gradle.org/distributions/gradle-5.2.1-all.zip
3
+ distributionBase=GRADLE_USER_HOME
4
+ distributionPath=wrapper/dists
5
+ zipStorePath=wrapper/dists
6
+ zipStoreBase=GRADLE_USER_HOME
data/gradlew ADDED
@@ -0,0 +1,172 @@
1
+ #!/usr/bin/env sh
2
+
3
+ ##############################################################################
4
+ ##
5
+ ## Gradle start up script for UN*X
6
+ ##
7
+ ##############################################################################
8
+
9
+ # Attempt to set APP_HOME
10
+ # Resolve links: $0 may be a link
11
+ PRG="$0"
12
+ # Need this for relative symlinks.
13
+ while [ -h "$PRG" ] ; do
14
+ ls=`ls -ld "$PRG"`
15
+ link=`expr "$ls" : '.*-> \(.*\)$'`
16
+ if expr "$link" : '/.*' > /dev/null; then
17
+ PRG="$link"
18
+ else
19
+ PRG=`dirname "$PRG"`"/$link"
20
+ fi
21
+ done
22
+ SAVED="`pwd`"
23
+ cd "`dirname \"$PRG\"`/" >/dev/null
24
+ APP_HOME="`pwd -P`"
25
+ cd "$SAVED" >/dev/null
26
+
27
+ APP_NAME="Gradle"
28
+ APP_BASE_NAME=`basename "$0"`
29
+
30
+ # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
31
+ DEFAULT_JVM_OPTS='"-Xmx64m"'
32
+
33
+ # Use the maximum available, or set MAX_FD != -1 to use that value.
34
+ MAX_FD="maximum"
35
+
36
+ warn () {
37
+ echo "$*"
38
+ }
39
+
40
+ die () {
41
+ echo
42
+ echo "$*"
43
+ echo
44
+ exit 1
45
+ }
46
+
47
+ # OS specific support (must be 'true' or 'false').
48
+ cygwin=false
49
+ msys=false
50
+ darwin=false
51
+ nonstop=false
52
+ case "`uname`" in
53
+ CYGWIN* )
54
+ cygwin=true
55
+ ;;
56
+ Darwin* )
57
+ darwin=true
58
+ ;;
59
+ MINGW* )
60
+ msys=true
61
+ ;;
62
+ NONSTOP* )
63
+ nonstop=true
64
+ ;;
65
+ esac
66
+
67
+ CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
68
+
69
+ # Determine the Java command to use to start the JVM.
70
+ if [ -n "$JAVA_HOME" ] ; then
71
+ if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
72
+ # IBM's JDK on AIX uses strange locations for the executables
73
+ JAVACMD="$JAVA_HOME/jre/sh/java"
74
+ else
75
+ JAVACMD="$JAVA_HOME/bin/java"
76
+ fi
77
+ if [ ! -x "$JAVACMD" ] ; then
78
+ die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
79
+
80
+ Please set the JAVA_HOME variable in your environment to match the
81
+ location of your Java installation."
82
+ fi
83
+ else
84
+ JAVACMD="java"
85
+ which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
86
+
87
+ Please set the JAVA_HOME variable in your environment to match the
88
+ location of your Java installation."
89
+ fi
90
+
91
+ # Increase the maximum file descriptors if we can.
92
+ if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
93
+ MAX_FD_LIMIT=`ulimit -H -n`
94
+ if [ $? -eq 0 ] ; then
95
+ if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
96
+ MAX_FD="$MAX_FD_LIMIT"
97
+ fi
98
+ ulimit -n $MAX_FD
99
+ if [ $? -ne 0 ] ; then
100
+ warn "Could not set maximum file descriptor limit: $MAX_FD"
101
+ fi
102
+ else
103
+ warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
104
+ fi
105
+ fi
106
+
107
+ # For Darwin, add options to specify how the application appears in the dock
108
+ if $darwin; then
109
+ GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
110
+ fi
111
+
112
+ # For Cygwin, switch paths to Windows format before running java
113
+ if $cygwin ; then
114
+ APP_HOME=`cygpath --path --mixed "$APP_HOME"`
115
+ CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
116
+ JAVACMD=`cygpath --unix "$JAVACMD"`
117
+
118
+ # We build the pattern for arguments to be converted via cygpath
119
+ ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
120
+ SEP=""
121
+ for dir in $ROOTDIRSRAW ; do
122
+ ROOTDIRS="$ROOTDIRS$SEP$dir"
123
+ SEP="|"
124
+ done
125
+ OURCYGPATTERN="(^($ROOTDIRS))"
126
+ # Add a user-defined pattern to the cygpath arguments
127
+ if [ "$GRADLE_CYGPATTERN" != "" ] ; then
128
+ OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
129
+ fi
130
+ # Now convert the arguments - kludge to limit ourselves to /bin/sh
131
+ i=0
132
+ for arg in "$@" ; do
133
+ CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
134
+ CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
135
+
136
+ if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
137
+ eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
138
+ else
139
+ eval `echo args$i`="\"$arg\""
140
+ fi
141
+ i=$((i+1))
142
+ done
143
+ case $i in
144
+ (0) set -- ;;
145
+ (1) set -- "$args0" ;;
146
+ (2) set -- "$args0" "$args1" ;;
147
+ (3) set -- "$args0" "$args1" "$args2" ;;
148
+ (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
149
+ (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
150
+ (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
151
+ (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
152
+ (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
153
+ (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
154
+ esac
155
+ fi
156
+
157
+ # Escape application args
158
+ save () {
159
+ for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
160
+ echo " "
161
+ }
162
+ APP_ARGS=$(save "$@")
163
+
164
+ # Collect all arguments for the java command, following the shell quoting and substitution rules
165
+ eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
166
+
167
+ # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
168
+ if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
169
+ cd "$(dirname "$0")"
170
+ fi
171
+
172
+ exec "$JAVACMD" "$@"
@@ -0,0 +1,84 @@
1
+ @if "%DEBUG%" == "" @echo off
2
+ @rem ##########################################################################
3
+ @rem
4
+ @rem Gradle startup script for Windows
5
+ @rem
6
+ @rem ##########################################################################
7
+
8
+ @rem Set local scope for the variables with windows NT shell
9
+ if "%OS%"=="Windows_NT" setlocal
10
+
11
+ set DIRNAME=%~dp0
12
+ if "%DIRNAME%" == "" set DIRNAME=.
13
+ set APP_BASE_NAME=%~n0
14
+ set APP_HOME=%DIRNAME%
15
+
16
+ @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
17
+ set DEFAULT_JVM_OPTS="-Xmx64m"
18
+
19
+ @rem Find java.exe
20
+ if defined JAVA_HOME goto findJavaFromJavaHome
21
+
22
+ set JAVA_EXE=java.exe
23
+ %JAVA_EXE% -version >NUL 2>&1
24
+ if "%ERRORLEVEL%" == "0" goto init
25
+
26
+ echo.
27
+ echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
28
+ echo.
29
+ echo Please set the JAVA_HOME variable in your environment to match the
30
+ echo location of your Java installation.
31
+
32
+ goto fail
33
+
34
+ :findJavaFromJavaHome
35
+ set JAVA_HOME=%JAVA_HOME:"=%
36
+ set JAVA_EXE=%JAVA_HOME%/bin/java.exe
37
+
38
+ if exist "%JAVA_EXE%" goto init
39
+
40
+ echo.
41
+ echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
42
+ echo.
43
+ echo Please set the JAVA_HOME variable in your environment to match the
44
+ echo location of your Java installation.
45
+
46
+ goto fail
47
+
48
+ :init
49
+ @rem Get command-line arguments, handling Windows variants
50
+
51
+ if not "%OS%" == "Windows_NT" goto win9xME_args
52
+
53
+ :win9xME_args
54
+ @rem Slurp the command line arguments.
55
+ set CMD_LINE_ARGS=
56
+ set _SKIP=2
57
+
58
+ :win9xME_args_slurp
59
+ if "x%~1" == "x" goto execute
60
+
61
+ set CMD_LINE_ARGS=%*
62
+
63
+ :execute
64
+ @rem Setup the command line
65
+
66
+ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
67
+
68
+ @rem Execute Gradle
69
+ "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
70
+
71
+ :end
72
+ @rem End local scope for the variables with windows NT shell
73
+ if "%ERRORLEVEL%"=="0" goto mainEnd
74
+
75
+ :fail
76
+ rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
77
+ rem the _cmd.exe /c_ return code!
78
+ if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
79
+ exit /b 1
80
+
81
+ :mainEnd
82
+ if "%OS%"=="Windows_NT" endlocal
83
+
84
+ :omega
@@ -0,0 +1,3 @@
1
+ Embulk::JavaPlugin.register_input(
2
+ "firestore", "org.embulk.input.firestore.FirestoreInputPlugin",
3
+ File.expand_path('../../../../classpath', __FILE__))
@@ -0,0 +1,102 @@
1
+ package org.embulk.input.firestore
2
+
3
+ import java.io.FileInputStream
4
+ import java.util
5
+
6
+ import com.fasterxml.jackson.databind.ObjectMapper
7
+ import com.google.auth.oauth2.GoogleCredentials
8
+ import com.google.cloud.firestore.Firestore
9
+ import com.google.firebase.cloud.FirestoreClient
10
+ import com.google.firebase.{FirebaseApp, FirebaseOptions}
11
+ import com.syucream.firesql.FireSQL
12
+ import org.embulk.config.{ConfigDiff, ConfigSource, TaskReport, TaskSource}
13
+ import org.embulk.spi.`type`.Types
14
+ import org.embulk.spi.PageBuilder
15
+ import org.embulk.spi.json.JsonParser
16
+ import org.embulk.spi.{Exec, InputPlugin, PageOutput, Schema}
17
+
18
+ import scala.util.Try
19
+
20
+ case class FirestoreInputPlugin() extends InputPlugin {
21
+ private val TASK_COUNT = 1
22
+
23
+ private val jsonParser = new JsonParser()
24
+ private val objectMapper = new ObjectMapper()
25
+
26
+ override def transaction(
27
+ config: ConfigSource,
28
+ control: InputPlugin.Control
29
+ ): ConfigDiff = {
30
+ val task = config.loadConfig(classOf[PluginTask])
31
+
32
+ // TODO runtime validations
33
+
34
+ val schema = Schema
35
+ .builder()
36
+ .add(task.getJsonColumnName, Types.JSON)
37
+ .build()
38
+
39
+ resume(task.dump(), schema, TASK_COUNT, control)
40
+ }
41
+
42
+ override def resume(
43
+ taskSource: TaskSource,
44
+ schema: Schema,
45
+ taskCount: Int,
46
+ control: InputPlugin.Control
47
+ ): ConfigDiff = {
48
+ // XXX unimplemented
49
+ control.run(taskSource, schema, taskCount)
50
+ Exec.newConfigDiff()
51
+ }
52
+
53
+ override def cleanup(
54
+ taskSource: TaskSource,
55
+ schema: Schema,
56
+ taskCount: Int,
57
+ successTaskReports: util.List[TaskReport]
58
+ ): Unit = {
59
+ // nothing to do
60
+ }
61
+
62
+ override def run(
63
+ taskSource: TaskSource,
64
+ schema: Schema,
65
+ taskIndex: Int,
66
+ output: PageOutput
67
+ ): TaskReport = {
68
+ val task = taskSource.loadTask(classOf[PluginTask])
69
+ val allocator = task.getBufferAllocator
70
+ val pageBuilder = new PageBuilder(allocator, schema, output)
71
+
72
+ val firestore = createFirestore(task.getJsonKeyfile).get
73
+ val firesql = new FireSQL(firestore)
74
+ val query = firesql.query(task.getSql)
75
+
76
+ val col = pageBuilder.getSchema.getColumn(0)
77
+
78
+ query.get.getDocuments.forEach { d =>
79
+ val json = objectMapper.writeValueAsString(d.getData)
80
+ pageBuilder.setJson(col, jsonParser.parse(json))
81
+ pageBuilder.addRecord()
82
+ }
83
+ pageBuilder.finish()
84
+
85
+ Exec.newTaskReport()
86
+ }
87
+
88
+ override def guess(config: ConfigSource): ConfigDiff =
89
+ Exec.newConfigDiff()
90
+
91
+ private def createFirestore(pathToCredJson: String): Try[Firestore] = {
92
+ Try {
93
+ val serviceAccount = new FileInputStream(pathToCredJson)
94
+ val credentials = GoogleCredentials.fromStream(serviceAccount)
95
+ val options =
96
+ new FirebaseOptions.Builder().setCredentials(credentials).build
97
+ FirebaseApp.initializeApp(options)
98
+
99
+ FirestoreClient.getFirestore
100
+ }
101
+ }
102
+ }
@@ -0,0 +1,23 @@
1
+ package org.embulk.input.firestore
2
+
3
+ import org.embulk.config.{Config, ConfigDefault, ConfigInject, Task}
4
+ import org.embulk.spi.BufferAllocator
5
+
6
+ trait PluginTask extends Task {
7
+
8
+ @Config("project_id")
9
+ def getProjectId: String
10
+
11
+ @Config("json_keyfile")
12
+ def getJsonKeyfile: String
13
+
14
+ @Config("json_column_name")
15
+ @ConfigDefault("\"record\"")
16
+ def getJsonColumnName: String
17
+
18
+ @Config("sql")
19
+ def getSql: String
20
+
21
+ @ConfigInject
22
+ def getBufferAllocator: BufferAllocator
23
+ }
metadata ADDED
@@ -0,0 +1,125 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: embulk-input-firestore
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Ryo Okubo
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2020-01-02 00:00:00.000000000 Z
12
+ dependencies: []
13
+ description: Selects records from a table.
14
+ email:
15
+ - syucream@gmail.com
16
+ executables: []
17
+ extensions: []
18
+ extra_rdoc_files: []
19
+ files:
20
+ - ".gitignore"
21
+ - ".scalafmt.conf"
22
+ - LICENSE
23
+ - README.md
24
+ - build.gradle
25
+ - classpath/animal-sniffer-annotations-1.17.jar
26
+ - classpath/annotations-4.1.1.4.jar
27
+ - classpath/api-common-1.8.1.jar
28
+ - classpath/auto-value-annotations-1.6.6.jar
29
+ - classpath/checker-qual-2.8.1.jar
30
+ - classpath/commons-codec-1.11.jar
31
+ - classpath/commons-lang3-3.5.jar
32
+ - classpath/commons-logging-1.2.jar
33
+ - classpath/commons-math3-3.6.1.jar
34
+ - classpath/embulk-input-firestore-0.0.1-shadow.jar
35
+ - classpath/error_prone_annotations-2.3.2.jar
36
+ - classpath/failureaccess-1.0.1.jar
37
+ - classpath/gax-1.48.1.jar
38
+ - classpath/gax-grpc-1.48.1.jar
39
+ - classpath/gax-httpjson-0.65.1.jar
40
+ - classpath/google-api-client-1.30.2.jar
41
+ - classpath/google-api-client-gson-1.30.1.jar
42
+ - classpath/google-api-services-storage-v1-rev20190624-1.30.1.jar
43
+ - classpath/google-auth-library-credentials-0.17.1.jar
44
+ - classpath/google-auth-library-oauth2-http-0.17.1.jar
45
+ - classpath/google-cloud-core-1.90.0.jar
46
+ - classpath/google-cloud-core-grpc-1.90.0.jar
47
+ - classpath/google-cloud-core-http-1.90.0.jar
48
+ - classpath/google-cloud-storage-1.91.0.jar
49
+ - classpath/google-http-client-1.31.0.jar
50
+ - classpath/google-http-client-appengine-1.31.0.jar
51
+ - classpath/google-http-client-gson-1.30.1.jar
52
+ - classpath/google-http-client-jackson2-1.31.0.jar
53
+ - classpath/google-oauth-client-1.30.1.jar
54
+ - classpath/grpc-alts-1.23.0.jar
55
+ - classpath/grpc-auth-1.23.0.jar
56
+ - classpath/grpc-context-1.23.0.jar
57
+ - classpath/grpc-grpclb-1.23.0.jar
58
+ - classpath/grpc-protobuf-1.23.0.jar
59
+ - classpath/grpc-protobuf-lite-1.23.0.jar
60
+ - classpath/grpc-stub-1.23.0.jar
61
+ - classpath/gson-2.8.5.jar
62
+ - classpath/httpclient-4.5.9.jar
63
+ - classpath/httpcore-4.4.11.jar
64
+ - classpath/j2objc-annotations-1.3.jar
65
+ - classpath/jackson-core-2.9.9.jar
66
+ - classpath/javax.annotation-api-1.3.2.jar
67
+ - classpath/jsqlparser-3.1.jar
68
+ - classpath/jsr305-3.0.2.jar
69
+ - classpath/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
70
+ - classpath/netty-buffer-4.1.34.Final.jar
71
+ - classpath/netty-codec-4.1.34.Final.jar
72
+ - classpath/netty-codec-http-4.1.34.Final.jar
73
+ - classpath/netty-common-4.1.34.Final.jar
74
+ - classpath/netty-handler-4.1.34.Final.jar
75
+ - classpath/netty-resolver-4.1.34.Final.jar
76
+ - classpath/netty-transport-4.1.34.Final.jar
77
+ - classpath/opencensus-api-0.23.0.jar
78
+ - classpath/opencensus-contrib-grpc-metrics-0.21.0.jar
79
+ - classpath/opencensus-contrib-grpc-util-0.23.0.jar
80
+ - classpath/opencensus-contrib-http-util-0.21.0.jar
81
+ - classpath/perfmark-api-0.17.0.jar
82
+ - classpath/proto-google-cloud-firestore-admin-v1-1.21.0.jar
83
+ - classpath/proto-google-cloud-firestore-v1-1.21.0.jar
84
+ - classpath/proto-google-cloud-firestore-v1beta1-0.74.0.jar
85
+ - classpath/proto-google-common-protos-1.16.0.jar
86
+ - classpath/proto-google-iam-v1-0.12.0.jar
87
+ - classpath/protobuf-java-3.9.1.jar
88
+ - classpath/protobuf-java-util-3.9.1.jar
89
+ - classpath/scala-library-2.12.3.jar
90
+ - classpath/slf4j-api-1.7.25.jar
91
+ - classpath/threetenbp-1.3.3.jar
92
+ - examples/firestore2stdout.yaml
93
+ - gradle.properties
94
+ - gradle/wrapper/gradle-wrapper.jar
95
+ - gradle/wrapper/gradle-wrapper.properties
96
+ - gradlew
97
+ - gradlew.bat
98
+ - lib/embulk/input/firestore.rb
99
+ - src/main/scala/org/embulk/input/firestore/FirestoreInputPlugin.scala
100
+ - src/main/scala/org/embulk/input/firestore/PluginTask.scala
101
+ homepage: https://github.com/syucream/embulk-input-firestore
102
+ licenses:
103
+ - Apache-2.0
104
+ metadata: {}
105
+ post_install_message:
106
+ rdoc_options: []
107
+ require_paths:
108
+ - lib
109
+ required_ruby_version: !ruby/object:Gem::Requirement
110
+ requirements:
111
+ - - ">="
112
+ - !ruby/object:Gem::Version
113
+ version: '0'
114
+ required_rubygems_version: !ruby/object:Gem::Requirement
115
+ requirements:
116
+ - - ">="
117
+ - !ruby/object:Gem::Version
118
+ version: '0'
119
+ requirements: []
120
+ rubyforge_project:
121
+ rubygems_version: 2.6.13
122
+ signing_key:
123
+ specification_version: 4
124
+ summary: Cloud Firestoreinput plugin for Embulk
125
+ test_files: []