embulk-output-td 0.2.2 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/build.gradle +9 -8
- data/embulk-output-td.gemspec +1 -1
- data/src/main/java/org/embulk/output/td/MsgpackGZFileBuilder.java +11 -12
- data/src/main/java/org/embulk/output/td/RecordWriter.java +4 -7
- data/src/main/java/org/embulk/output/td/TdOutputPlugin.java +89 -84
- data/src/main/java/org/embulk/output/td/writer/FieldWriterSet.java +9 -0
- data/src/main/java/org/embulk/output/td/writer/JsonFieldWriter.java +23 -0
- data/src/test/java/org/embulk/output/td/TestRecordWriter.java +37 -38
- data/src/test/java/org/embulk/output/td/TestTdOutputPlugin.java +53 -49
- metadata +9 -30
- data/src/main/java/com/treasuredata/api/TdApiClient.java +0 -506
- data/src/main/java/com/treasuredata/api/TdApiClientConfig.java +0 -79
- data/src/main/java/com/treasuredata/api/TdApiConflictException.java +0 -10
- data/src/main/java/com/treasuredata/api/TdApiConstants.java +0 -10
- data/src/main/java/com/treasuredata/api/TdApiException.java +0 -20
- data/src/main/java/com/treasuredata/api/TdApiExecutionException.java +0 -10
- data/src/main/java/com/treasuredata/api/TdApiExecutionInterruptedException.java +0 -16
- data/src/main/java/com/treasuredata/api/TdApiExecutionTimeoutException.java +0 -18
- data/src/main/java/com/treasuredata/api/TdApiNotFoundException.java +0 -10
- data/src/main/java/com/treasuredata/api/TdApiResponseException.java +0 -32
- data/src/main/java/com/treasuredata/api/model/TDArrayColumnType.java +0 -80
- data/src/main/java/com/treasuredata/api/model/TDBulkImportSession.java +0 -157
- data/src/main/java/com/treasuredata/api/model/TDColumn.java +0 -129
- data/src/main/java/com/treasuredata/api/model/TDColumnType.java +0 -23
- data/src/main/java/com/treasuredata/api/model/TDColumnTypeDeserializer.java +0 -128
- data/src/main/java/com/treasuredata/api/model/TDDatabase.java +0 -49
- data/src/main/java/com/treasuredata/api/model/TDDatabaseList.java +0 -24
- data/src/main/java/com/treasuredata/api/model/TDMapColumnType.java +0 -88
- data/src/main/java/com/treasuredata/api/model/TDPrimitiveColumnType.java +0 -61
- data/src/main/java/com/treasuredata/api/model/TDTable.java +0 -64
- data/src/main/java/com/treasuredata/api/model/TDTableList.java +0 -33
- data/src/main/java/com/treasuredata/api/model/TDTablePermission.java +0 -50
- data/src/main/java/com/treasuredata/api/model/TDTableSchema.java +0 -44
- data/src/main/java/com/treasuredata/api/model/TDTableType.java +0 -37
- data/src/test/java/com/treasuredata/api/TestTdApiClient.java +0 -79
@@ -1,79 +0,0 @@
|
|
1
|
-
package com.treasuredata.api;
|
2
|
-
|
3
|
-
import com.google.common.base.Optional;
|
4
|
-
|
5
|
-
public class TdApiClientConfig
|
6
|
-
{
|
7
|
-
public static class HttpProxyConfig
|
8
|
-
{
|
9
|
-
private String host;
|
10
|
-
private int port;
|
11
|
-
private boolean secure;
|
12
|
-
|
13
|
-
public HttpProxyConfig(String host, int port, boolean secure)
|
14
|
-
{
|
15
|
-
this.host = host;
|
16
|
-
this.port = port;
|
17
|
-
this.secure = secure;
|
18
|
-
}
|
19
|
-
|
20
|
-
public String getHost()
|
21
|
-
{
|
22
|
-
return host;
|
23
|
-
}
|
24
|
-
|
25
|
-
public int getPort()
|
26
|
-
{
|
27
|
-
return port;
|
28
|
-
}
|
29
|
-
|
30
|
-
public boolean isSecure()
|
31
|
-
{
|
32
|
-
return secure;
|
33
|
-
}
|
34
|
-
}
|
35
|
-
|
36
|
-
private String endpoint;
|
37
|
-
private boolean useSsl;
|
38
|
-
private Optional<HttpProxyConfig> httpProxyConfig;
|
39
|
-
|
40
|
-
// TODO Builder
|
41
|
-
// TODO clone
|
42
|
-
|
43
|
-
public TdApiClientConfig(String endpoint, boolean useSsl)
|
44
|
-
{
|
45
|
-
this(endpoint, useSsl, Optional.<HttpProxyConfig>absent());
|
46
|
-
}
|
47
|
-
|
48
|
-
public TdApiClientConfig(String endpoint, boolean useSsl, Optional<HttpProxyConfig> httpProxyConfig)
|
49
|
-
{
|
50
|
-
this.endpoint = endpoint;
|
51
|
-
this.useSsl = useSsl;
|
52
|
-
this.httpProxyConfig = httpProxyConfig;
|
53
|
-
}
|
54
|
-
|
55
|
-
public String getEndpoint()
|
56
|
-
{
|
57
|
-
return endpoint;
|
58
|
-
}
|
59
|
-
|
60
|
-
public void setEndpoint(String endpoint)
|
61
|
-
{
|
62
|
-
this.endpoint = endpoint;
|
63
|
-
}
|
64
|
-
|
65
|
-
public boolean getUseSsl()
|
66
|
-
{
|
67
|
-
return useSsl;
|
68
|
-
}
|
69
|
-
|
70
|
-
public Optional<HttpProxyConfig> getHttpProxyConfig()
|
71
|
-
{
|
72
|
-
return httpProxyConfig;
|
73
|
-
}
|
74
|
-
|
75
|
-
public String getAgentName()
|
76
|
-
{
|
77
|
-
return TdApiConstants.AGENT_NAME;
|
78
|
-
}
|
79
|
-
}
|
@@ -1,20 +0,0 @@
|
|
1
|
-
package com.treasuredata.api;
|
2
|
-
|
3
|
-
public class TdApiException
|
4
|
-
extends RuntimeException
|
5
|
-
{
|
6
|
-
public TdApiException(String message)
|
7
|
-
{
|
8
|
-
super(message);
|
9
|
-
}
|
10
|
-
|
11
|
-
public TdApiException(Throwable cause)
|
12
|
-
{
|
13
|
-
super(cause);
|
14
|
-
}
|
15
|
-
|
16
|
-
public TdApiException(String message, Throwable cause)
|
17
|
-
{
|
18
|
-
super(message, cause);
|
19
|
-
}
|
20
|
-
}
|
@@ -1,16 +0,0 @@
|
|
1
|
-
package com.treasuredata.api;
|
2
|
-
|
3
|
-
public class TdApiExecutionInterruptedException
|
4
|
-
extends TdApiExecutionException
|
5
|
-
{
|
6
|
-
public TdApiExecutionInterruptedException(InterruptedException cause)
|
7
|
-
{
|
8
|
-
super(cause);
|
9
|
-
}
|
10
|
-
|
11
|
-
@Override
|
12
|
-
public InterruptedException getCause()
|
13
|
-
{
|
14
|
-
return (InterruptedException) super.getCause();
|
15
|
-
}
|
16
|
-
}
|
@@ -1,18 +0,0 @@
|
|
1
|
-
package com.treasuredata.api;
|
2
|
-
|
3
|
-
import java.util.concurrent.TimeoutException;
|
4
|
-
|
5
|
-
public class TdApiExecutionTimeoutException
|
6
|
-
extends TdApiExecutionException
|
7
|
-
{
|
8
|
-
public TdApiExecutionTimeoutException(TimeoutException cause)
|
9
|
-
{
|
10
|
-
super(cause);
|
11
|
-
}
|
12
|
-
|
13
|
-
@Override
|
14
|
-
public TimeoutException getCause()
|
15
|
-
{
|
16
|
-
return (TimeoutException) super.getCause();
|
17
|
-
}
|
18
|
-
}
|
@@ -1,32 +0,0 @@
|
|
1
|
-
package com.treasuredata.api;
|
2
|
-
|
3
|
-
public class TdApiResponseException
|
4
|
-
extends TdApiException
|
5
|
-
{
|
6
|
-
private final int status;
|
7
|
-
private final byte[] body;
|
8
|
-
|
9
|
-
public TdApiResponseException(int status, byte[] body)
|
10
|
-
{
|
11
|
-
super(String.format("TD API returned HTTP code %d", status));
|
12
|
-
this.status = status;
|
13
|
-
this.body = body;
|
14
|
-
}
|
15
|
-
|
16
|
-
public TdApiResponseException(int status, byte[] body, Throwable cause)
|
17
|
-
{
|
18
|
-
super(String.format("TD API returned HTTP code %d", status), cause);
|
19
|
-
this.status = status;
|
20
|
-
this.body = body;
|
21
|
-
}
|
22
|
-
|
23
|
-
public int getStatusCode()
|
24
|
-
{
|
25
|
-
return status;
|
26
|
-
}
|
27
|
-
|
28
|
-
public byte[] getBody()
|
29
|
-
{
|
30
|
-
return body;
|
31
|
-
}
|
32
|
-
}
|
@@ -1,80 +0,0 @@
|
|
1
|
-
package com.treasuredata.api.model;
|
2
|
-
|
3
|
-
import com.google.common.base.Objects;
|
4
|
-
|
5
|
-
public class TDArrayColumnType
|
6
|
-
implements TDColumnType
|
7
|
-
{
|
8
|
-
private final TDColumnType elementType;
|
9
|
-
|
10
|
-
public TDArrayColumnType(TDColumnType elementType)
|
11
|
-
{
|
12
|
-
this.elementType = elementType;
|
13
|
-
}
|
14
|
-
|
15
|
-
public TDColumnType getElementType()
|
16
|
-
{
|
17
|
-
return elementType;
|
18
|
-
}
|
19
|
-
|
20
|
-
@Override
|
21
|
-
public String toString()
|
22
|
-
{
|
23
|
-
return "array<" + elementType + ">";
|
24
|
-
}
|
25
|
-
|
26
|
-
@Override
|
27
|
-
public boolean isPrimitive()
|
28
|
-
{
|
29
|
-
return false;
|
30
|
-
}
|
31
|
-
|
32
|
-
@Override
|
33
|
-
public boolean isArrayType()
|
34
|
-
{
|
35
|
-
return true;
|
36
|
-
}
|
37
|
-
|
38
|
-
@Override
|
39
|
-
public boolean isMapType()
|
40
|
-
{
|
41
|
-
return false;
|
42
|
-
}
|
43
|
-
|
44
|
-
@Override
|
45
|
-
public TDPrimitiveColumnType asPrimitiveType()
|
46
|
-
{
|
47
|
-
return null;
|
48
|
-
}
|
49
|
-
|
50
|
-
@Override
|
51
|
-
public TDArrayColumnType asArrayType()
|
52
|
-
{
|
53
|
-
return this;
|
54
|
-
}
|
55
|
-
|
56
|
-
@Override
|
57
|
-
public TDMapColumnType asMapType()
|
58
|
-
{
|
59
|
-
return null;
|
60
|
-
}
|
61
|
-
|
62
|
-
@Override
|
63
|
-
public boolean equals(Object obj)
|
64
|
-
{
|
65
|
-
if (this == obj) {
|
66
|
-
return true;
|
67
|
-
}
|
68
|
-
if (obj == null || getClass() != obj.getClass()) {
|
69
|
-
return false;
|
70
|
-
}
|
71
|
-
TDArrayColumnType other = (TDArrayColumnType) obj;
|
72
|
-
return Objects.equal(this.elementType, other.elementType);
|
73
|
-
}
|
74
|
-
|
75
|
-
@Override
|
76
|
-
public int hashCode()
|
77
|
-
{
|
78
|
-
return Objects.hashCode(elementType);
|
79
|
-
}
|
80
|
-
}
|
@@ -1,157 +0,0 @@
|
|
1
|
-
package com.treasuredata.api.model;
|
2
|
-
|
3
|
-
import com.fasterxml.jackson.annotation.JsonCreator;
|
4
|
-
import com.fasterxml.jackson.annotation.JsonProperty;
|
5
|
-
|
6
|
-
public class TDBulkImportSession
|
7
|
-
{
|
8
|
-
public static enum ImportStatus
|
9
|
-
{
|
10
|
-
UPLOADING("uploading"),
|
11
|
-
PERFORMING("performing"),
|
12
|
-
READY("ready"),
|
13
|
-
COMMITTING("committing"),
|
14
|
-
COMMITTED("committed"),
|
15
|
-
UNKNOWN("unknown");
|
16
|
-
|
17
|
-
private final String name;
|
18
|
-
|
19
|
-
private ImportStatus(String name)
|
20
|
-
{
|
21
|
-
this.name = name;
|
22
|
-
}
|
23
|
-
|
24
|
-
@JsonCreator
|
25
|
-
public static ImportStatus forName(String name)
|
26
|
-
{
|
27
|
-
return ImportStatus.valueOf(name.toUpperCase());
|
28
|
-
}
|
29
|
-
}
|
30
|
-
|
31
|
-
private final String name;
|
32
|
-
private final String databaseName;
|
33
|
-
private final String tableName;
|
34
|
-
|
35
|
-
private final ImportStatus status;
|
36
|
-
|
37
|
-
private final boolean uploadFrozen;
|
38
|
-
private final String jobId; // nullable
|
39
|
-
private final long validRecords;
|
40
|
-
private final long errorRecords;
|
41
|
-
private final long validParts;
|
42
|
-
private final long errorParts;
|
43
|
-
|
44
|
-
@JsonCreator
|
45
|
-
public TDBulkImportSession(
|
46
|
-
@JsonProperty("name") String name,
|
47
|
-
@JsonProperty("database") String databaseName,
|
48
|
-
@JsonProperty("table") String tableName,
|
49
|
-
@JsonProperty("status") ImportStatus status,
|
50
|
-
@JsonProperty("upload_frozen") boolean uploadFrozen,
|
51
|
-
@JsonProperty("job_id") String jobId,
|
52
|
-
@JsonProperty("valid_records") long validRecords,
|
53
|
-
@JsonProperty("error_records") long errorRecords,
|
54
|
-
@JsonProperty("valid_parts") long validParts,
|
55
|
-
@JsonProperty("error_parts") long errorParts)
|
56
|
-
{
|
57
|
-
this.name = name;
|
58
|
-
this.databaseName = databaseName;
|
59
|
-
this.tableName = tableName;
|
60
|
-
this.status = status;
|
61
|
-
this.uploadFrozen = uploadFrozen;
|
62
|
-
this.jobId = jobId;
|
63
|
-
this.validRecords = validRecords;
|
64
|
-
this.errorRecords = errorRecords;
|
65
|
-
this.validParts = validParts;
|
66
|
-
this.errorParts = errorParts;
|
67
|
-
}
|
68
|
-
|
69
|
-
@JsonProperty
|
70
|
-
public String getName()
|
71
|
-
{
|
72
|
-
return name;
|
73
|
-
}
|
74
|
-
|
75
|
-
@JsonProperty("database")
|
76
|
-
public String getDatabaseName()
|
77
|
-
{
|
78
|
-
return databaseName;
|
79
|
-
}
|
80
|
-
|
81
|
-
@JsonProperty("table")
|
82
|
-
public String getTableName()
|
83
|
-
{
|
84
|
-
return tableName;
|
85
|
-
}
|
86
|
-
|
87
|
-
@JsonProperty
|
88
|
-
public ImportStatus getStatus()
|
89
|
-
{
|
90
|
-
return status;
|
91
|
-
}
|
92
|
-
|
93
|
-
@JsonProperty("upload_frozen")
|
94
|
-
public boolean getUploadFrozen()
|
95
|
-
{
|
96
|
-
return uploadFrozen;
|
97
|
-
}
|
98
|
-
|
99
|
-
@JsonProperty("job_id")
|
100
|
-
public String getJobId()
|
101
|
-
{
|
102
|
-
return jobId;
|
103
|
-
}
|
104
|
-
|
105
|
-
@JsonProperty("valid_records")
|
106
|
-
public long getValidRecords()
|
107
|
-
{
|
108
|
-
return validRecords;
|
109
|
-
}
|
110
|
-
|
111
|
-
@JsonProperty("error_records")
|
112
|
-
public long getErrorRecords()
|
113
|
-
{
|
114
|
-
return errorRecords;
|
115
|
-
}
|
116
|
-
|
117
|
-
@JsonProperty("valid_parts")
|
118
|
-
public long getValidParts()
|
119
|
-
{
|
120
|
-
return validParts;
|
121
|
-
}
|
122
|
-
|
123
|
-
@JsonProperty("error_parts")
|
124
|
-
public long getErrorParts()
|
125
|
-
{
|
126
|
-
return errorParts;
|
127
|
-
}
|
128
|
-
|
129
|
-
public boolean isUploading()
|
130
|
-
{
|
131
|
-
return status == ImportStatus.UPLOADING;
|
132
|
-
}
|
133
|
-
|
134
|
-
public boolean is(ImportStatus expecting)
|
135
|
-
{
|
136
|
-
return status == expecting;
|
137
|
-
}
|
138
|
-
|
139
|
-
public boolean isPeformError()
|
140
|
-
{
|
141
|
-
return validRecords == 0 || errorParts > 0 || errorRecords > 0;
|
142
|
-
}
|
143
|
-
|
144
|
-
public String getErrorMessage()
|
145
|
-
{
|
146
|
-
if (validRecords == 0) {
|
147
|
-
return "No record processed";
|
148
|
-
}
|
149
|
-
if (errorRecords > 0) {
|
150
|
-
return String.format("%d invalid parts", errorParts);
|
151
|
-
}
|
152
|
-
if (errorRecords > 0) {
|
153
|
-
return String.format("%d invalid records", errorRecords);
|
154
|
-
}
|
155
|
-
return null;
|
156
|
-
}
|
157
|
-
}
|