embulk-input-firestore 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +2 -0
- data/.scalafmt.conf +13 -0
- data/LICENSE +21 -0
- data/README.md +2 -0
- data/build.gradle +91 -0
- data/classpath/animal-sniffer-annotations-1.17.jar +0 -0
- data/classpath/annotations-4.1.1.4.jar +0 -0
- data/classpath/api-common-1.8.1.jar +0 -0
- data/classpath/auto-value-annotations-1.6.6.jar +0 -0
- data/classpath/checker-qual-2.8.1.jar +0 -0
- data/classpath/commons-codec-1.11.jar +0 -0
- data/classpath/commons-lang3-3.5.jar +0 -0
- data/classpath/commons-logging-1.2.jar +0 -0
- data/classpath/commons-math3-3.6.1.jar +0 -0
- data/classpath/embulk-input-firestore-0.0.1-shadow.jar +0 -0
- data/classpath/error_prone_annotations-2.3.2.jar +0 -0
- data/classpath/failureaccess-1.0.1.jar +0 -0
- data/classpath/gax-1.48.1.jar +0 -0
- data/classpath/gax-grpc-1.48.1.jar +0 -0
- data/classpath/gax-httpjson-0.65.1.jar +0 -0
- data/classpath/google-api-client-1.30.2.jar +0 -0
- data/classpath/google-api-client-gson-1.30.1.jar +0 -0
- data/classpath/google-api-services-storage-v1-rev20190624-1.30.1.jar +0 -0
- data/classpath/google-auth-library-credentials-0.17.1.jar +0 -0
- data/classpath/google-auth-library-oauth2-http-0.17.1.jar +0 -0
- data/classpath/google-cloud-core-1.90.0.jar +0 -0
- data/classpath/google-cloud-core-grpc-1.90.0.jar +0 -0
- data/classpath/google-cloud-core-http-1.90.0.jar +0 -0
- data/classpath/google-cloud-storage-1.91.0.jar +0 -0
- data/classpath/google-http-client-1.31.0.jar +0 -0
- data/classpath/google-http-client-appengine-1.31.0.jar +0 -0
- data/classpath/google-http-client-gson-1.30.1.jar +0 -0
- data/classpath/google-http-client-jackson2-1.31.0.jar +0 -0
- data/classpath/google-oauth-client-1.30.1.jar +0 -0
- data/classpath/grpc-alts-1.23.0.jar +0 -0
- data/classpath/grpc-auth-1.23.0.jar +0 -0
- data/classpath/grpc-context-1.23.0.jar +0 -0
- data/classpath/grpc-grpclb-1.23.0.jar +0 -0
- data/classpath/grpc-protobuf-1.23.0.jar +0 -0
- data/classpath/grpc-protobuf-lite-1.23.0.jar +0 -0
- data/classpath/grpc-stub-1.23.0.jar +0 -0
- data/classpath/gson-2.8.5.jar +0 -0
- data/classpath/httpclient-4.5.9.jar +0 -0
- data/classpath/httpcore-4.4.11.jar +0 -0
- data/classpath/j2objc-annotations-1.3.jar +0 -0
- data/classpath/jackson-core-2.9.9.jar +0 -0
- data/classpath/javax.annotation-api-1.3.2.jar +0 -0
- data/classpath/jsqlparser-3.1.jar +0 -0
- data/classpath/jsr305-3.0.2.jar +0 -0
- data/classpath/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar +0 -0
- data/classpath/netty-buffer-4.1.34.Final.jar +0 -0
- data/classpath/netty-codec-4.1.34.Final.jar +0 -0
- data/classpath/netty-codec-http-4.1.34.Final.jar +0 -0
- data/classpath/netty-common-4.1.34.Final.jar +0 -0
- data/classpath/netty-handler-4.1.34.Final.jar +0 -0
- data/classpath/netty-resolver-4.1.34.Final.jar +0 -0
- data/classpath/netty-transport-4.1.34.Final.jar +0 -0
- data/classpath/opencensus-api-0.23.0.jar +0 -0
- data/classpath/opencensus-contrib-grpc-metrics-0.21.0.jar +0 -0
- data/classpath/opencensus-contrib-grpc-util-0.23.0.jar +0 -0
- data/classpath/opencensus-contrib-http-util-0.21.0.jar +0 -0
- data/classpath/perfmark-api-0.17.0.jar +0 -0
- data/classpath/proto-google-cloud-firestore-admin-v1-1.21.0.jar +0 -0
- data/classpath/proto-google-cloud-firestore-v1-1.21.0.jar +0 -0
- data/classpath/proto-google-cloud-firestore-v1beta1-0.74.0.jar +0 -0
- data/classpath/proto-google-common-protos-1.16.0.jar +0 -0
- data/classpath/proto-google-iam-v1-0.12.0.jar +0 -0
- data/classpath/protobuf-java-3.9.1.jar +0 -0
- data/classpath/protobuf-java-util-3.9.1.jar +0 -0
- data/classpath/scala-library-2.12.3.jar +0 -0
- data/classpath/slf4j-api-1.7.25.jar +0 -0
- data/classpath/threetenbp-1.3.3.jar +0 -0
- data/examples/firestore2stdout.yaml +8 -0
- data/gradle.properties +1 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +6 -0
- data/gradlew +172 -0
- data/gradlew.bat +84 -0
- data/lib/embulk/input/firestore.rb +3 -0
- data/src/main/scala/org/embulk/input/firestore/FirestoreInputPlugin.scala +102 -0
- data/src/main/scala/org/embulk/input/firestore/PluginTask.scala +23 -0
- metadata +125 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: fee292d3a8ec98d9fd54b54f164a4f6d3a081266f60317fc928d51277a48c7a2
|
4
|
+
data.tar.gz: fc811f4a0d4f84210d474b3d576aa7e1553b818de3298d86dbb3ec7b365844a8
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: d66db9159e3609029ca3eabb1fc79a3ed85cd936a1ca38f41d33e7fdec5ce5a14b301bdc2fa3d465cbee939f3cab3cde9ac86d5432252823d56daad5c088966d
|
7
|
+
data.tar.gz: 02d238fe94eb849fc1639b3312509ba6ebd1a239aea22f9d939471daadddb9cdfd95fe596656f88513c47d02590a22ac9b8142f0426895f46dd7cd861ac81f32
|
data/.gitignore
ADDED
data/.scalafmt.conf
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
version=2.3.2
|
2
|
+
project.git = true
|
3
|
+
project.excludeFilters = [
|
4
|
+
scalafmt-benchmarks/src/resources,
|
5
|
+
sbt-test
|
6
|
+
bin/issue
|
7
|
+
]
|
8
|
+
align = none
|
9
|
+
# Disabled in default since this operation is potentially
|
10
|
+
# dangerous if you define your own stripMargin with different
|
11
|
+
# semantics from the stdlib stripMargin.
|
12
|
+
assumeStandardLibraryStripMargin = true
|
13
|
+
onTestFailure = "To fix this, run ./scalafmt from the project root directory"
|
data/LICENSE
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
MIT License
|
2
|
+
|
3
|
+
Copyright (c) 2019 Ryo Okubo
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
13
|
+
copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
data/build.gradle
ADDED
@@ -0,0 +1,91 @@
|
|
1
|
+
plugins {
|
2
|
+
id 'scala'
|
3
|
+
id 'com.github.jruby-gradle.base' version '1.6.0'
|
4
|
+
id 'com.github.johnrengelman.shadow' version '5.2.0'
|
5
|
+
id 'cz.alenkacz.gradle.scalafmt' version '1.10.0'
|
6
|
+
}
|
7
|
+
import com.github.jrubygradle.JRubyExec
|
8
|
+
|
9
|
+
configurations {
|
10
|
+
provided
|
11
|
+
}
|
12
|
+
|
13
|
+
repositories {
|
14
|
+
mavenCentral()
|
15
|
+
mavenLocal()
|
16
|
+
jcenter()
|
17
|
+
}
|
18
|
+
|
19
|
+
// Relocate Guava packages since it's incompatible with Guava's version from Embulk
|
20
|
+
shadowJar {
|
21
|
+
classifier 'shadow'
|
22
|
+
|
23
|
+
dependencies {
|
24
|
+
include dependency('com.google.guava:guava')
|
25
|
+
|
26
|
+
include dependency('com.google.cloud:google-cloud-firestore')
|
27
|
+
include dependency('com.google.firebase:firebase-admin')
|
28
|
+
include dependency('com.syucream:firesql')
|
29
|
+
|
30
|
+
include dependency('io.grpc:grpc-netty-shaded')
|
31
|
+
include dependency('io.grpc:grpc-core')
|
32
|
+
include dependency('io.grpc:grpc-api')
|
33
|
+
}
|
34
|
+
|
35
|
+
relocate 'com.google.common', 'relocated.com.google.common'
|
36
|
+
}
|
37
|
+
|
38
|
+
sourceCompatibility = 1.8
|
39
|
+
targetCompatibility = 1.8
|
40
|
+
|
41
|
+
dependencies {
|
42
|
+
compile 'org.scala-lang:scala-library:2.12.3'
|
43
|
+
|
44
|
+
compile 'org.embulk:embulk-core:0.9.12'
|
45
|
+
provided 'org.embulk:embulk-core:0.9.12'
|
46
|
+
|
47
|
+
// Should be shadowed
|
48
|
+
compile 'com.google.firebase:firebase-admin:6.11.0'
|
49
|
+
compile 'com.syucream:firesql:0.0.1'
|
50
|
+
}
|
51
|
+
|
52
|
+
task classpath(type: Copy, dependsOn: ['jar', 'shadowJar']) {
|
53
|
+
doFirst { file('classpath').deleteDir() }
|
54
|
+
|
55
|
+
from (configurations.runtime
|
56
|
+
- configurations.provided
|
57
|
+
+ configurations.shadow
|
58
|
+
- files(shadowJar.getIncludedDependencies())
|
59
|
+
+ files(shadowJar.archiveFile))
|
60
|
+
|
61
|
+
into 'classpath'
|
62
|
+
}
|
63
|
+
clean { delete 'classpath' }
|
64
|
+
|
65
|
+
task gem(type: JRubyExec, dependsOn: ['build', 'gemspec', 'classpath']) {
|
66
|
+
script 'gem'
|
67
|
+
scriptArgs 'build', 'build/gemspec'
|
68
|
+
doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: 'pkg') }
|
69
|
+
}
|
70
|
+
|
71
|
+
task gemspec {
|
72
|
+
doLast {
|
73
|
+
file('build').mkdirs()
|
74
|
+
file('build/gemspec').write($/
|
75
|
+
Gem::Specification.new do |spec|
|
76
|
+
spec.name = "${project.name}"
|
77
|
+
spec.version = "${project.version}"
|
78
|
+
spec.authors = ["Ryo Okubo"]
|
79
|
+
spec.summary = %[Cloud Firestoreinput plugin for Embulk]
|
80
|
+
spec.description = %[Selects records from a table.]
|
81
|
+
spec.email = ["syucream@gmail.com"]
|
82
|
+
spec.licenses = ["Apache-2.0"]
|
83
|
+
spec.homepage = "https://github.com/syucream/embulk-input-firestore"
|
84
|
+
|
85
|
+
spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
|
86
|
+
spec.test_files = spec.files.grep(%r"^(test|spec)/")
|
87
|
+
spec.require_paths = ["lib"]
|
88
|
+
end
|
89
|
+
/$)
|
90
|
+
}
|
91
|
+
}
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
data/gradle.properties
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
version=0.0.1
|
Binary file
|
data/gradlew
ADDED
@@ -0,0 +1,172 @@
|
|
1
|
+
#!/usr/bin/env sh
|
2
|
+
|
3
|
+
##############################################################################
|
4
|
+
##
|
5
|
+
## Gradle start up script for UN*X
|
6
|
+
##
|
7
|
+
##############################################################################
|
8
|
+
|
9
|
+
# Attempt to set APP_HOME
|
10
|
+
# Resolve links: $0 may be a link
|
11
|
+
PRG="$0"
|
12
|
+
# Need this for relative symlinks.
|
13
|
+
while [ -h "$PRG" ] ; do
|
14
|
+
ls=`ls -ld "$PRG"`
|
15
|
+
link=`expr "$ls" : '.*-> \(.*\)$'`
|
16
|
+
if expr "$link" : '/.*' > /dev/null; then
|
17
|
+
PRG="$link"
|
18
|
+
else
|
19
|
+
PRG=`dirname "$PRG"`"/$link"
|
20
|
+
fi
|
21
|
+
done
|
22
|
+
SAVED="`pwd`"
|
23
|
+
cd "`dirname \"$PRG\"`/" >/dev/null
|
24
|
+
APP_HOME="`pwd -P`"
|
25
|
+
cd "$SAVED" >/dev/null
|
26
|
+
|
27
|
+
APP_NAME="Gradle"
|
28
|
+
APP_BASE_NAME=`basename "$0"`
|
29
|
+
|
30
|
+
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
31
|
+
DEFAULT_JVM_OPTS='"-Xmx64m"'
|
32
|
+
|
33
|
+
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
34
|
+
MAX_FD="maximum"
|
35
|
+
|
36
|
+
warn () {
|
37
|
+
echo "$*"
|
38
|
+
}
|
39
|
+
|
40
|
+
die () {
|
41
|
+
echo
|
42
|
+
echo "$*"
|
43
|
+
echo
|
44
|
+
exit 1
|
45
|
+
}
|
46
|
+
|
47
|
+
# OS specific support (must be 'true' or 'false').
|
48
|
+
cygwin=false
|
49
|
+
msys=false
|
50
|
+
darwin=false
|
51
|
+
nonstop=false
|
52
|
+
case "`uname`" in
|
53
|
+
CYGWIN* )
|
54
|
+
cygwin=true
|
55
|
+
;;
|
56
|
+
Darwin* )
|
57
|
+
darwin=true
|
58
|
+
;;
|
59
|
+
MINGW* )
|
60
|
+
msys=true
|
61
|
+
;;
|
62
|
+
NONSTOP* )
|
63
|
+
nonstop=true
|
64
|
+
;;
|
65
|
+
esac
|
66
|
+
|
67
|
+
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
|
68
|
+
|
69
|
+
# Determine the Java command to use to start the JVM.
|
70
|
+
if [ -n "$JAVA_HOME" ] ; then
|
71
|
+
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
|
72
|
+
# IBM's JDK on AIX uses strange locations for the executables
|
73
|
+
JAVACMD="$JAVA_HOME/jre/sh/java"
|
74
|
+
else
|
75
|
+
JAVACMD="$JAVA_HOME/bin/java"
|
76
|
+
fi
|
77
|
+
if [ ! -x "$JAVACMD" ] ; then
|
78
|
+
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
|
79
|
+
|
80
|
+
Please set the JAVA_HOME variable in your environment to match the
|
81
|
+
location of your Java installation."
|
82
|
+
fi
|
83
|
+
else
|
84
|
+
JAVACMD="java"
|
85
|
+
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
86
|
+
|
87
|
+
Please set the JAVA_HOME variable in your environment to match the
|
88
|
+
location of your Java installation."
|
89
|
+
fi
|
90
|
+
|
91
|
+
# Increase the maximum file descriptors if we can.
|
92
|
+
if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
|
93
|
+
MAX_FD_LIMIT=`ulimit -H -n`
|
94
|
+
if [ $? -eq 0 ] ; then
|
95
|
+
if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
|
96
|
+
MAX_FD="$MAX_FD_LIMIT"
|
97
|
+
fi
|
98
|
+
ulimit -n $MAX_FD
|
99
|
+
if [ $? -ne 0 ] ; then
|
100
|
+
warn "Could not set maximum file descriptor limit: $MAX_FD"
|
101
|
+
fi
|
102
|
+
else
|
103
|
+
warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
|
104
|
+
fi
|
105
|
+
fi
|
106
|
+
|
107
|
+
# For Darwin, add options to specify how the application appears in the dock
|
108
|
+
if $darwin; then
|
109
|
+
GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
|
110
|
+
fi
|
111
|
+
|
112
|
+
# For Cygwin, switch paths to Windows format before running java
|
113
|
+
if $cygwin ; then
|
114
|
+
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
|
115
|
+
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
|
116
|
+
JAVACMD=`cygpath --unix "$JAVACMD"`
|
117
|
+
|
118
|
+
# We build the pattern for arguments to be converted via cygpath
|
119
|
+
ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
|
120
|
+
SEP=""
|
121
|
+
for dir in $ROOTDIRSRAW ; do
|
122
|
+
ROOTDIRS="$ROOTDIRS$SEP$dir"
|
123
|
+
SEP="|"
|
124
|
+
done
|
125
|
+
OURCYGPATTERN="(^($ROOTDIRS))"
|
126
|
+
# Add a user-defined pattern to the cygpath arguments
|
127
|
+
if [ "$GRADLE_CYGPATTERN" != "" ] ; then
|
128
|
+
OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
|
129
|
+
fi
|
130
|
+
# Now convert the arguments - kludge to limit ourselves to /bin/sh
|
131
|
+
i=0
|
132
|
+
for arg in "$@" ; do
|
133
|
+
CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
|
134
|
+
CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
|
135
|
+
|
136
|
+
if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
|
137
|
+
eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
|
138
|
+
else
|
139
|
+
eval `echo args$i`="\"$arg\""
|
140
|
+
fi
|
141
|
+
i=$((i+1))
|
142
|
+
done
|
143
|
+
case $i in
|
144
|
+
(0) set -- ;;
|
145
|
+
(1) set -- "$args0" ;;
|
146
|
+
(2) set -- "$args0" "$args1" ;;
|
147
|
+
(3) set -- "$args0" "$args1" "$args2" ;;
|
148
|
+
(4) set -- "$args0" "$args1" "$args2" "$args3" ;;
|
149
|
+
(5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
|
150
|
+
(6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
|
151
|
+
(7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
|
152
|
+
(8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
|
153
|
+
(9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
|
154
|
+
esac
|
155
|
+
fi
|
156
|
+
|
157
|
+
# Escape application args
|
158
|
+
save () {
|
159
|
+
for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
|
160
|
+
echo " "
|
161
|
+
}
|
162
|
+
APP_ARGS=$(save "$@")
|
163
|
+
|
164
|
+
# Collect all arguments for the java command, following the shell quoting and substitution rules
|
165
|
+
eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
|
166
|
+
|
167
|
+
# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
|
168
|
+
if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
|
169
|
+
cd "$(dirname "$0")"
|
170
|
+
fi
|
171
|
+
|
172
|
+
exec "$JAVACMD" "$@"
|
data/gradlew.bat
ADDED
@@ -0,0 +1,84 @@
|
|
1
|
+
@if "%DEBUG%" == "" @echo off
|
2
|
+
@rem ##########################################################################
|
3
|
+
@rem
|
4
|
+
@rem Gradle startup script for Windows
|
5
|
+
@rem
|
6
|
+
@rem ##########################################################################
|
7
|
+
|
8
|
+
@rem Set local scope for the variables with windows NT shell
|
9
|
+
if "%OS%"=="Windows_NT" setlocal
|
10
|
+
|
11
|
+
set DIRNAME=%~dp0
|
12
|
+
if "%DIRNAME%" == "" set DIRNAME=.
|
13
|
+
set APP_BASE_NAME=%~n0
|
14
|
+
set APP_HOME=%DIRNAME%
|
15
|
+
|
16
|
+
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
17
|
+
set DEFAULT_JVM_OPTS="-Xmx64m"
|
18
|
+
|
19
|
+
@rem Find java.exe
|
20
|
+
if defined JAVA_HOME goto findJavaFromJavaHome
|
21
|
+
|
22
|
+
set JAVA_EXE=java.exe
|
23
|
+
%JAVA_EXE% -version >NUL 2>&1
|
24
|
+
if "%ERRORLEVEL%" == "0" goto init
|
25
|
+
|
26
|
+
echo.
|
27
|
+
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
28
|
+
echo.
|
29
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
30
|
+
echo location of your Java installation.
|
31
|
+
|
32
|
+
goto fail
|
33
|
+
|
34
|
+
:findJavaFromJavaHome
|
35
|
+
set JAVA_HOME=%JAVA_HOME:"=%
|
36
|
+
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
37
|
+
|
38
|
+
if exist "%JAVA_EXE%" goto init
|
39
|
+
|
40
|
+
echo.
|
41
|
+
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
42
|
+
echo.
|
43
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
44
|
+
echo location of your Java installation.
|
45
|
+
|
46
|
+
goto fail
|
47
|
+
|
48
|
+
:init
|
49
|
+
@rem Get command-line arguments, handling Windows variants
|
50
|
+
|
51
|
+
if not "%OS%" == "Windows_NT" goto win9xME_args
|
52
|
+
|
53
|
+
:win9xME_args
|
54
|
+
@rem Slurp the command line arguments.
|
55
|
+
set CMD_LINE_ARGS=
|
56
|
+
set _SKIP=2
|
57
|
+
|
58
|
+
:win9xME_args_slurp
|
59
|
+
if "x%~1" == "x" goto execute
|
60
|
+
|
61
|
+
set CMD_LINE_ARGS=%*
|
62
|
+
|
63
|
+
:execute
|
64
|
+
@rem Setup the command line
|
65
|
+
|
66
|
+
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
67
|
+
|
68
|
+
@rem Execute Gradle
|
69
|
+
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
|
70
|
+
|
71
|
+
:end
|
72
|
+
@rem End local scope for the variables with windows NT shell
|
73
|
+
if "%ERRORLEVEL%"=="0" goto mainEnd
|
74
|
+
|
75
|
+
:fail
|
76
|
+
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
77
|
+
rem the _cmd.exe /c_ return code!
|
78
|
+
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
79
|
+
exit /b 1
|
80
|
+
|
81
|
+
:mainEnd
|
82
|
+
if "%OS%"=="Windows_NT" endlocal
|
83
|
+
|
84
|
+
:omega
|
@@ -0,0 +1,102 @@
|
|
1
|
+
package org.embulk.input.firestore
|
2
|
+
|
3
|
+
import java.io.FileInputStream
|
4
|
+
import java.util
|
5
|
+
|
6
|
+
import com.fasterxml.jackson.databind.ObjectMapper
|
7
|
+
import com.google.auth.oauth2.GoogleCredentials
|
8
|
+
import com.google.cloud.firestore.Firestore
|
9
|
+
import com.google.firebase.cloud.FirestoreClient
|
10
|
+
import com.google.firebase.{FirebaseApp, FirebaseOptions}
|
11
|
+
import com.syucream.firesql.FireSQL
|
12
|
+
import org.embulk.config.{ConfigDiff, ConfigSource, TaskReport, TaskSource}
|
13
|
+
import org.embulk.spi.`type`.Types
|
14
|
+
import org.embulk.spi.PageBuilder
|
15
|
+
import org.embulk.spi.json.JsonParser
|
16
|
+
import org.embulk.spi.{Exec, InputPlugin, PageOutput, Schema}
|
17
|
+
|
18
|
+
import scala.util.Try
|
19
|
+
|
20
|
+
case class FirestoreInputPlugin() extends InputPlugin {
|
21
|
+
private val TASK_COUNT = 1
|
22
|
+
|
23
|
+
private val jsonParser = new JsonParser()
|
24
|
+
private val objectMapper = new ObjectMapper()
|
25
|
+
|
26
|
+
override def transaction(
|
27
|
+
config: ConfigSource,
|
28
|
+
control: InputPlugin.Control
|
29
|
+
): ConfigDiff = {
|
30
|
+
val task = config.loadConfig(classOf[PluginTask])
|
31
|
+
|
32
|
+
// TODO runtime validations
|
33
|
+
|
34
|
+
val schema = Schema
|
35
|
+
.builder()
|
36
|
+
.add(task.getJsonColumnName, Types.JSON)
|
37
|
+
.build()
|
38
|
+
|
39
|
+
resume(task.dump(), schema, TASK_COUNT, control)
|
40
|
+
}
|
41
|
+
|
42
|
+
override def resume(
|
43
|
+
taskSource: TaskSource,
|
44
|
+
schema: Schema,
|
45
|
+
taskCount: Int,
|
46
|
+
control: InputPlugin.Control
|
47
|
+
): ConfigDiff = {
|
48
|
+
// XXX unimplemented
|
49
|
+
control.run(taskSource, schema, taskCount)
|
50
|
+
Exec.newConfigDiff()
|
51
|
+
}
|
52
|
+
|
53
|
+
override def cleanup(
|
54
|
+
taskSource: TaskSource,
|
55
|
+
schema: Schema,
|
56
|
+
taskCount: Int,
|
57
|
+
successTaskReports: util.List[TaskReport]
|
58
|
+
): Unit = {
|
59
|
+
// nothing to do
|
60
|
+
}
|
61
|
+
|
62
|
+
override def run(
|
63
|
+
taskSource: TaskSource,
|
64
|
+
schema: Schema,
|
65
|
+
taskIndex: Int,
|
66
|
+
output: PageOutput
|
67
|
+
): TaskReport = {
|
68
|
+
val task = taskSource.loadTask(classOf[PluginTask])
|
69
|
+
val allocator = task.getBufferAllocator
|
70
|
+
val pageBuilder = new PageBuilder(allocator, schema, output)
|
71
|
+
|
72
|
+
val firestore = createFirestore(task.getJsonKeyfile).get
|
73
|
+
val firesql = new FireSQL(firestore)
|
74
|
+
val query = firesql.query(task.getSql)
|
75
|
+
|
76
|
+
val col = pageBuilder.getSchema.getColumn(0)
|
77
|
+
|
78
|
+
query.get.getDocuments.forEach { d =>
|
79
|
+
val json = objectMapper.writeValueAsString(d.getData)
|
80
|
+
pageBuilder.setJson(col, jsonParser.parse(json))
|
81
|
+
pageBuilder.addRecord()
|
82
|
+
}
|
83
|
+
pageBuilder.finish()
|
84
|
+
|
85
|
+
Exec.newTaskReport()
|
86
|
+
}
|
87
|
+
|
88
|
+
override def guess(config: ConfigSource): ConfigDiff =
|
89
|
+
Exec.newConfigDiff()
|
90
|
+
|
91
|
+
private def createFirestore(pathToCredJson: String): Try[Firestore] = {
|
92
|
+
Try {
|
93
|
+
val serviceAccount = new FileInputStream(pathToCredJson)
|
94
|
+
val credentials = GoogleCredentials.fromStream(serviceAccount)
|
95
|
+
val options =
|
96
|
+
new FirebaseOptions.Builder().setCredentials(credentials).build
|
97
|
+
FirebaseApp.initializeApp(options)
|
98
|
+
|
99
|
+
FirestoreClient.getFirestore
|
100
|
+
}
|
101
|
+
}
|
102
|
+
}
|
@@ -0,0 +1,23 @@
|
|
1
|
+
package org.embulk.input.firestore
|
2
|
+
|
3
|
+
import org.embulk.config.{Config, ConfigDefault, ConfigInject, Task}
|
4
|
+
import org.embulk.spi.BufferAllocator
|
5
|
+
|
6
|
+
trait PluginTask extends Task {
|
7
|
+
|
8
|
+
@Config("project_id")
|
9
|
+
def getProjectId: String
|
10
|
+
|
11
|
+
@Config("json_keyfile")
|
12
|
+
def getJsonKeyfile: String
|
13
|
+
|
14
|
+
@Config("json_column_name")
|
15
|
+
@ConfigDefault("\"record\"")
|
16
|
+
def getJsonColumnName: String
|
17
|
+
|
18
|
+
@Config("sql")
|
19
|
+
def getSql: String
|
20
|
+
|
21
|
+
@ConfigInject
|
22
|
+
def getBufferAllocator: BufferAllocator
|
23
|
+
}
|
metadata
ADDED
@@ -0,0 +1,125 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: embulk-input-firestore
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Ryo Okubo
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2020-01-02 00:00:00.000000000 Z
|
12
|
+
dependencies: []
|
13
|
+
description: Selects records from a table.
|
14
|
+
email:
|
15
|
+
- syucream@gmail.com
|
16
|
+
executables: []
|
17
|
+
extensions: []
|
18
|
+
extra_rdoc_files: []
|
19
|
+
files:
|
20
|
+
- ".gitignore"
|
21
|
+
- ".scalafmt.conf"
|
22
|
+
- LICENSE
|
23
|
+
- README.md
|
24
|
+
- build.gradle
|
25
|
+
- classpath/animal-sniffer-annotations-1.17.jar
|
26
|
+
- classpath/annotations-4.1.1.4.jar
|
27
|
+
- classpath/api-common-1.8.1.jar
|
28
|
+
- classpath/auto-value-annotations-1.6.6.jar
|
29
|
+
- classpath/checker-qual-2.8.1.jar
|
30
|
+
- classpath/commons-codec-1.11.jar
|
31
|
+
- classpath/commons-lang3-3.5.jar
|
32
|
+
- classpath/commons-logging-1.2.jar
|
33
|
+
- classpath/commons-math3-3.6.1.jar
|
34
|
+
- classpath/embulk-input-firestore-0.0.1-shadow.jar
|
35
|
+
- classpath/error_prone_annotations-2.3.2.jar
|
36
|
+
- classpath/failureaccess-1.0.1.jar
|
37
|
+
- classpath/gax-1.48.1.jar
|
38
|
+
- classpath/gax-grpc-1.48.1.jar
|
39
|
+
- classpath/gax-httpjson-0.65.1.jar
|
40
|
+
- classpath/google-api-client-1.30.2.jar
|
41
|
+
- classpath/google-api-client-gson-1.30.1.jar
|
42
|
+
- classpath/google-api-services-storage-v1-rev20190624-1.30.1.jar
|
43
|
+
- classpath/google-auth-library-credentials-0.17.1.jar
|
44
|
+
- classpath/google-auth-library-oauth2-http-0.17.1.jar
|
45
|
+
- classpath/google-cloud-core-1.90.0.jar
|
46
|
+
- classpath/google-cloud-core-grpc-1.90.0.jar
|
47
|
+
- classpath/google-cloud-core-http-1.90.0.jar
|
48
|
+
- classpath/google-cloud-storage-1.91.0.jar
|
49
|
+
- classpath/google-http-client-1.31.0.jar
|
50
|
+
- classpath/google-http-client-appengine-1.31.0.jar
|
51
|
+
- classpath/google-http-client-gson-1.30.1.jar
|
52
|
+
- classpath/google-http-client-jackson2-1.31.0.jar
|
53
|
+
- classpath/google-oauth-client-1.30.1.jar
|
54
|
+
- classpath/grpc-alts-1.23.0.jar
|
55
|
+
- classpath/grpc-auth-1.23.0.jar
|
56
|
+
- classpath/grpc-context-1.23.0.jar
|
57
|
+
- classpath/grpc-grpclb-1.23.0.jar
|
58
|
+
- classpath/grpc-protobuf-1.23.0.jar
|
59
|
+
- classpath/grpc-protobuf-lite-1.23.0.jar
|
60
|
+
- classpath/grpc-stub-1.23.0.jar
|
61
|
+
- classpath/gson-2.8.5.jar
|
62
|
+
- classpath/httpclient-4.5.9.jar
|
63
|
+
- classpath/httpcore-4.4.11.jar
|
64
|
+
- classpath/j2objc-annotations-1.3.jar
|
65
|
+
- classpath/jackson-core-2.9.9.jar
|
66
|
+
- classpath/javax.annotation-api-1.3.2.jar
|
67
|
+
- classpath/jsqlparser-3.1.jar
|
68
|
+
- classpath/jsr305-3.0.2.jar
|
69
|
+
- classpath/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
|
70
|
+
- classpath/netty-buffer-4.1.34.Final.jar
|
71
|
+
- classpath/netty-codec-4.1.34.Final.jar
|
72
|
+
- classpath/netty-codec-http-4.1.34.Final.jar
|
73
|
+
- classpath/netty-common-4.1.34.Final.jar
|
74
|
+
- classpath/netty-handler-4.1.34.Final.jar
|
75
|
+
- classpath/netty-resolver-4.1.34.Final.jar
|
76
|
+
- classpath/netty-transport-4.1.34.Final.jar
|
77
|
+
- classpath/opencensus-api-0.23.0.jar
|
78
|
+
- classpath/opencensus-contrib-grpc-metrics-0.21.0.jar
|
79
|
+
- classpath/opencensus-contrib-grpc-util-0.23.0.jar
|
80
|
+
- classpath/opencensus-contrib-http-util-0.21.0.jar
|
81
|
+
- classpath/perfmark-api-0.17.0.jar
|
82
|
+
- classpath/proto-google-cloud-firestore-admin-v1-1.21.0.jar
|
83
|
+
- classpath/proto-google-cloud-firestore-v1-1.21.0.jar
|
84
|
+
- classpath/proto-google-cloud-firestore-v1beta1-0.74.0.jar
|
85
|
+
- classpath/proto-google-common-protos-1.16.0.jar
|
86
|
+
- classpath/proto-google-iam-v1-0.12.0.jar
|
87
|
+
- classpath/protobuf-java-3.9.1.jar
|
88
|
+
- classpath/protobuf-java-util-3.9.1.jar
|
89
|
+
- classpath/scala-library-2.12.3.jar
|
90
|
+
- classpath/slf4j-api-1.7.25.jar
|
91
|
+
- classpath/threetenbp-1.3.3.jar
|
92
|
+
- examples/firestore2stdout.yaml
|
93
|
+
- gradle.properties
|
94
|
+
- gradle/wrapper/gradle-wrapper.jar
|
95
|
+
- gradle/wrapper/gradle-wrapper.properties
|
96
|
+
- gradlew
|
97
|
+
- gradlew.bat
|
98
|
+
- lib/embulk/input/firestore.rb
|
99
|
+
- src/main/scala/org/embulk/input/firestore/FirestoreInputPlugin.scala
|
100
|
+
- src/main/scala/org/embulk/input/firestore/PluginTask.scala
|
101
|
+
homepage: https://github.com/syucream/embulk-input-firestore
|
102
|
+
licenses:
|
103
|
+
- Apache-2.0
|
104
|
+
metadata: {}
|
105
|
+
post_install_message:
|
106
|
+
rdoc_options: []
|
107
|
+
require_paths:
|
108
|
+
- lib
|
109
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
110
|
+
requirements:
|
111
|
+
- - ">="
|
112
|
+
- !ruby/object:Gem::Version
|
113
|
+
version: '0'
|
114
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
115
|
+
requirements:
|
116
|
+
- - ">="
|
117
|
+
- !ruby/object:Gem::Version
|
118
|
+
version: '0'
|
119
|
+
requirements: []
|
120
|
+
rubyforge_project:
|
121
|
+
rubygems_version: 2.6.13
|
122
|
+
signing_key:
|
123
|
+
specification_version: 4
|
124
|
+
summary: Cloud Firestoreinput plugin for Embulk
|
125
|
+
test_files: []
|