embulk-output-td 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/build.gradle +9 -8
- data/embulk-output-td.gemspec +1 -1
- data/src/main/java/org/embulk/output/td/MsgpackGZFileBuilder.java +11 -12
- data/src/main/java/org/embulk/output/td/RecordWriter.java +4 -7
- data/src/main/java/org/embulk/output/td/TdOutputPlugin.java +89 -84
- data/src/main/java/org/embulk/output/td/writer/FieldWriterSet.java +9 -0
- data/src/main/java/org/embulk/output/td/writer/JsonFieldWriter.java +23 -0
- data/src/test/java/org/embulk/output/td/TestRecordWriter.java +37 -38
- data/src/test/java/org/embulk/output/td/TestTdOutputPlugin.java +53 -49
- metadata +9 -30
- data/src/main/java/com/treasuredata/api/TdApiClient.java +0 -506
- data/src/main/java/com/treasuredata/api/TdApiClientConfig.java +0 -79
- data/src/main/java/com/treasuredata/api/TdApiConflictException.java +0 -10
- data/src/main/java/com/treasuredata/api/TdApiConstants.java +0 -10
- data/src/main/java/com/treasuredata/api/TdApiException.java +0 -20
- data/src/main/java/com/treasuredata/api/TdApiExecutionException.java +0 -10
- data/src/main/java/com/treasuredata/api/TdApiExecutionInterruptedException.java +0 -16
- data/src/main/java/com/treasuredata/api/TdApiExecutionTimeoutException.java +0 -18
- data/src/main/java/com/treasuredata/api/TdApiNotFoundException.java +0 -10
- data/src/main/java/com/treasuredata/api/TdApiResponseException.java +0 -32
- data/src/main/java/com/treasuredata/api/model/TDArrayColumnType.java +0 -80
- data/src/main/java/com/treasuredata/api/model/TDBulkImportSession.java +0 -157
- data/src/main/java/com/treasuredata/api/model/TDColumn.java +0 -129
- data/src/main/java/com/treasuredata/api/model/TDColumnType.java +0 -23
- data/src/main/java/com/treasuredata/api/model/TDColumnTypeDeserializer.java +0 -128
- data/src/main/java/com/treasuredata/api/model/TDDatabase.java +0 -49
- data/src/main/java/com/treasuredata/api/model/TDDatabaseList.java +0 -24
- data/src/main/java/com/treasuredata/api/model/TDMapColumnType.java +0 -88
- data/src/main/java/com/treasuredata/api/model/TDPrimitiveColumnType.java +0 -61
- data/src/main/java/com/treasuredata/api/model/TDTable.java +0 -64
- data/src/main/java/com/treasuredata/api/model/TDTableList.java +0 -33
- data/src/main/java/com/treasuredata/api/model/TDTablePermission.java +0 -50
- data/src/main/java/com/treasuredata/api/model/TDTableSchema.java +0 -44
- data/src/main/java/com/treasuredata/api/model/TDTableType.java +0 -37
- data/src/test/java/com/treasuredata/api/TestTdApiClient.java +0 -79
@@ -1,79 +0,0 @@
|
|
1
|
-
package com.treasuredata.api;
|
2
|
-
|
3
|
-
import com.google.common.base.Optional;
|
4
|
-
|
5
|
-
public class TdApiClientConfig
|
6
|
-
{
|
7
|
-
public static class HttpProxyConfig
|
8
|
-
{
|
9
|
-
private String host;
|
10
|
-
private int port;
|
11
|
-
private boolean secure;
|
12
|
-
|
13
|
-
public HttpProxyConfig(String host, int port, boolean secure)
|
14
|
-
{
|
15
|
-
this.host = host;
|
16
|
-
this.port = port;
|
17
|
-
this.secure = secure;
|
18
|
-
}
|
19
|
-
|
20
|
-
public String getHost()
|
21
|
-
{
|
22
|
-
return host;
|
23
|
-
}
|
24
|
-
|
25
|
-
public int getPort()
|
26
|
-
{
|
27
|
-
return port;
|
28
|
-
}
|
29
|
-
|
30
|
-
public boolean isSecure()
|
31
|
-
{
|
32
|
-
return secure;
|
33
|
-
}
|
34
|
-
}
|
35
|
-
|
36
|
-
private String endpoint;
|
37
|
-
private boolean useSsl;
|
38
|
-
private Optional<HttpProxyConfig> httpProxyConfig;
|
39
|
-
|
40
|
-
// TODO Builder
|
41
|
-
// TODO clone
|
42
|
-
|
43
|
-
public TdApiClientConfig(String endpoint, boolean useSsl)
|
44
|
-
{
|
45
|
-
this(endpoint, useSsl, Optional.<HttpProxyConfig>absent());
|
46
|
-
}
|
47
|
-
|
48
|
-
public TdApiClientConfig(String endpoint, boolean useSsl, Optional<HttpProxyConfig> httpProxyConfig)
|
49
|
-
{
|
50
|
-
this.endpoint = endpoint;
|
51
|
-
this.useSsl = useSsl;
|
52
|
-
this.httpProxyConfig = httpProxyConfig;
|
53
|
-
}
|
54
|
-
|
55
|
-
public String getEndpoint()
|
56
|
-
{
|
57
|
-
return endpoint;
|
58
|
-
}
|
59
|
-
|
60
|
-
public void setEndpoint(String endpoint)
|
61
|
-
{
|
62
|
-
this.endpoint = endpoint;
|
63
|
-
}
|
64
|
-
|
65
|
-
public boolean getUseSsl()
|
66
|
-
{
|
67
|
-
return useSsl;
|
68
|
-
}
|
69
|
-
|
70
|
-
public Optional<HttpProxyConfig> getHttpProxyConfig()
|
71
|
-
{
|
72
|
-
return httpProxyConfig;
|
73
|
-
}
|
74
|
-
|
75
|
-
public String getAgentName()
|
76
|
-
{
|
77
|
-
return TdApiConstants.AGENT_NAME;
|
78
|
-
}
|
79
|
-
}
|
@@ -1,20 +0,0 @@
|
|
1
|
-
package com.treasuredata.api;
|
2
|
-
|
3
|
-
public class TdApiException
|
4
|
-
extends RuntimeException
|
5
|
-
{
|
6
|
-
public TdApiException(String message)
|
7
|
-
{
|
8
|
-
super(message);
|
9
|
-
}
|
10
|
-
|
11
|
-
public TdApiException(Throwable cause)
|
12
|
-
{
|
13
|
-
super(cause);
|
14
|
-
}
|
15
|
-
|
16
|
-
public TdApiException(String message, Throwable cause)
|
17
|
-
{
|
18
|
-
super(message, cause);
|
19
|
-
}
|
20
|
-
}
|
@@ -1,16 +0,0 @@
|
|
1
|
-
package com.treasuredata.api;
|
2
|
-
|
3
|
-
public class TdApiExecutionInterruptedException
|
4
|
-
extends TdApiExecutionException
|
5
|
-
{
|
6
|
-
public TdApiExecutionInterruptedException(InterruptedException cause)
|
7
|
-
{
|
8
|
-
super(cause);
|
9
|
-
}
|
10
|
-
|
11
|
-
@Override
|
12
|
-
public InterruptedException getCause()
|
13
|
-
{
|
14
|
-
return (InterruptedException) super.getCause();
|
15
|
-
}
|
16
|
-
}
|
@@ -1,18 +0,0 @@
|
|
1
|
-
package com.treasuredata.api;
|
2
|
-
|
3
|
-
import java.util.concurrent.TimeoutException;
|
4
|
-
|
5
|
-
public class TdApiExecutionTimeoutException
|
6
|
-
extends TdApiExecutionException
|
7
|
-
{
|
8
|
-
public TdApiExecutionTimeoutException(TimeoutException cause)
|
9
|
-
{
|
10
|
-
super(cause);
|
11
|
-
}
|
12
|
-
|
13
|
-
@Override
|
14
|
-
public TimeoutException getCause()
|
15
|
-
{
|
16
|
-
return (TimeoutException) super.getCause();
|
17
|
-
}
|
18
|
-
}
|
@@ -1,32 +0,0 @@
|
|
1
|
-
package com.treasuredata.api;
|
2
|
-
|
3
|
-
public class TdApiResponseException
|
4
|
-
extends TdApiException
|
5
|
-
{
|
6
|
-
private final int status;
|
7
|
-
private final byte[] body;
|
8
|
-
|
9
|
-
public TdApiResponseException(int status, byte[] body)
|
10
|
-
{
|
11
|
-
super(String.format("TD API returned HTTP code %d", status));
|
12
|
-
this.status = status;
|
13
|
-
this.body = body;
|
14
|
-
}
|
15
|
-
|
16
|
-
public TdApiResponseException(int status, byte[] body, Throwable cause)
|
17
|
-
{
|
18
|
-
super(String.format("TD API returned HTTP code %d", status), cause);
|
19
|
-
this.status = status;
|
20
|
-
this.body = body;
|
21
|
-
}
|
22
|
-
|
23
|
-
public int getStatusCode()
|
24
|
-
{
|
25
|
-
return status;
|
26
|
-
}
|
27
|
-
|
28
|
-
public byte[] getBody()
|
29
|
-
{
|
30
|
-
return body;
|
31
|
-
}
|
32
|
-
}
|
@@ -1,80 +0,0 @@
|
|
1
|
-
package com.treasuredata.api.model;
|
2
|
-
|
3
|
-
import com.google.common.base.Objects;
|
4
|
-
|
5
|
-
public class TDArrayColumnType
|
6
|
-
implements TDColumnType
|
7
|
-
{
|
8
|
-
private final TDColumnType elementType;
|
9
|
-
|
10
|
-
public TDArrayColumnType(TDColumnType elementType)
|
11
|
-
{
|
12
|
-
this.elementType = elementType;
|
13
|
-
}
|
14
|
-
|
15
|
-
public TDColumnType getElementType()
|
16
|
-
{
|
17
|
-
return elementType;
|
18
|
-
}
|
19
|
-
|
20
|
-
@Override
|
21
|
-
public String toString()
|
22
|
-
{
|
23
|
-
return "array<" + elementType + ">";
|
24
|
-
}
|
25
|
-
|
26
|
-
@Override
|
27
|
-
public boolean isPrimitive()
|
28
|
-
{
|
29
|
-
return false;
|
30
|
-
}
|
31
|
-
|
32
|
-
@Override
|
33
|
-
public boolean isArrayType()
|
34
|
-
{
|
35
|
-
return true;
|
36
|
-
}
|
37
|
-
|
38
|
-
@Override
|
39
|
-
public boolean isMapType()
|
40
|
-
{
|
41
|
-
return false;
|
42
|
-
}
|
43
|
-
|
44
|
-
@Override
|
45
|
-
public TDPrimitiveColumnType asPrimitiveType()
|
46
|
-
{
|
47
|
-
return null;
|
48
|
-
}
|
49
|
-
|
50
|
-
@Override
|
51
|
-
public TDArrayColumnType asArrayType()
|
52
|
-
{
|
53
|
-
return this;
|
54
|
-
}
|
55
|
-
|
56
|
-
@Override
|
57
|
-
public TDMapColumnType asMapType()
|
58
|
-
{
|
59
|
-
return null;
|
60
|
-
}
|
61
|
-
|
62
|
-
@Override
|
63
|
-
public boolean equals(Object obj)
|
64
|
-
{
|
65
|
-
if (this == obj) {
|
66
|
-
return true;
|
67
|
-
}
|
68
|
-
if (obj == null || getClass() != obj.getClass()) {
|
69
|
-
return false;
|
70
|
-
}
|
71
|
-
TDArrayColumnType other = (TDArrayColumnType) obj;
|
72
|
-
return Objects.equal(this.elementType, other.elementType);
|
73
|
-
}
|
74
|
-
|
75
|
-
@Override
|
76
|
-
public int hashCode()
|
77
|
-
{
|
78
|
-
return Objects.hashCode(elementType);
|
79
|
-
}
|
80
|
-
}
|
@@ -1,157 +0,0 @@
|
|
1
|
-
package com.treasuredata.api.model;
|
2
|
-
|
3
|
-
import com.fasterxml.jackson.annotation.JsonCreator;
|
4
|
-
import com.fasterxml.jackson.annotation.JsonProperty;
|
5
|
-
|
6
|
-
public class TDBulkImportSession
|
7
|
-
{
|
8
|
-
public static enum ImportStatus
|
9
|
-
{
|
10
|
-
UPLOADING("uploading"),
|
11
|
-
PERFORMING("performing"),
|
12
|
-
READY("ready"),
|
13
|
-
COMMITTING("committing"),
|
14
|
-
COMMITTED("committed"),
|
15
|
-
UNKNOWN("unknown");
|
16
|
-
|
17
|
-
private final String name;
|
18
|
-
|
19
|
-
private ImportStatus(String name)
|
20
|
-
{
|
21
|
-
this.name = name;
|
22
|
-
}
|
23
|
-
|
24
|
-
@JsonCreator
|
25
|
-
public static ImportStatus forName(String name)
|
26
|
-
{
|
27
|
-
return ImportStatus.valueOf(name.toUpperCase());
|
28
|
-
}
|
29
|
-
}
|
30
|
-
|
31
|
-
private final String name;
|
32
|
-
private final String databaseName;
|
33
|
-
private final String tableName;
|
34
|
-
|
35
|
-
private final ImportStatus status;
|
36
|
-
|
37
|
-
private final boolean uploadFrozen;
|
38
|
-
private final String jobId; // nullable
|
39
|
-
private final long validRecords;
|
40
|
-
private final long errorRecords;
|
41
|
-
private final long validParts;
|
42
|
-
private final long errorParts;
|
43
|
-
|
44
|
-
@JsonCreator
|
45
|
-
public TDBulkImportSession(
|
46
|
-
@JsonProperty("name") String name,
|
47
|
-
@JsonProperty("database") String databaseName,
|
48
|
-
@JsonProperty("table") String tableName,
|
49
|
-
@JsonProperty("status") ImportStatus status,
|
50
|
-
@JsonProperty("upload_frozen") boolean uploadFrozen,
|
51
|
-
@JsonProperty("job_id") String jobId,
|
52
|
-
@JsonProperty("valid_records") long validRecords,
|
53
|
-
@JsonProperty("error_records") long errorRecords,
|
54
|
-
@JsonProperty("valid_parts") long validParts,
|
55
|
-
@JsonProperty("error_parts") long errorParts)
|
56
|
-
{
|
57
|
-
this.name = name;
|
58
|
-
this.databaseName = databaseName;
|
59
|
-
this.tableName = tableName;
|
60
|
-
this.status = status;
|
61
|
-
this.uploadFrozen = uploadFrozen;
|
62
|
-
this.jobId = jobId;
|
63
|
-
this.validRecords = validRecords;
|
64
|
-
this.errorRecords = errorRecords;
|
65
|
-
this.validParts = validParts;
|
66
|
-
this.errorParts = errorParts;
|
67
|
-
}
|
68
|
-
|
69
|
-
@JsonProperty
|
70
|
-
public String getName()
|
71
|
-
{
|
72
|
-
return name;
|
73
|
-
}
|
74
|
-
|
75
|
-
@JsonProperty("database")
|
76
|
-
public String getDatabaseName()
|
77
|
-
{
|
78
|
-
return databaseName;
|
79
|
-
}
|
80
|
-
|
81
|
-
@JsonProperty("table")
|
82
|
-
public String getTableName()
|
83
|
-
{
|
84
|
-
return tableName;
|
85
|
-
}
|
86
|
-
|
87
|
-
@JsonProperty
|
88
|
-
public ImportStatus getStatus()
|
89
|
-
{
|
90
|
-
return status;
|
91
|
-
}
|
92
|
-
|
93
|
-
@JsonProperty("upload_frozen")
|
94
|
-
public boolean getUploadFrozen()
|
95
|
-
{
|
96
|
-
return uploadFrozen;
|
97
|
-
}
|
98
|
-
|
99
|
-
@JsonProperty("job_id")
|
100
|
-
public String getJobId()
|
101
|
-
{
|
102
|
-
return jobId;
|
103
|
-
}
|
104
|
-
|
105
|
-
@JsonProperty("valid_records")
|
106
|
-
public long getValidRecords()
|
107
|
-
{
|
108
|
-
return validRecords;
|
109
|
-
}
|
110
|
-
|
111
|
-
@JsonProperty("error_records")
|
112
|
-
public long getErrorRecords()
|
113
|
-
{
|
114
|
-
return errorRecords;
|
115
|
-
}
|
116
|
-
|
117
|
-
@JsonProperty("valid_parts")
|
118
|
-
public long getValidParts()
|
119
|
-
{
|
120
|
-
return validParts;
|
121
|
-
}
|
122
|
-
|
123
|
-
@JsonProperty("error_parts")
|
124
|
-
public long getErrorParts()
|
125
|
-
{
|
126
|
-
return errorParts;
|
127
|
-
}
|
128
|
-
|
129
|
-
public boolean isUploading()
|
130
|
-
{
|
131
|
-
return status == ImportStatus.UPLOADING;
|
132
|
-
}
|
133
|
-
|
134
|
-
public boolean is(ImportStatus expecting)
|
135
|
-
{
|
136
|
-
return status == expecting;
|
137
|
-
}
|
138
|
-
|
139
|
-
public boolean isPeformError()
|
140
|
-
{
|
141
|
-
return validRecords == 0 || errorParts > 0 || errorRecords > 0;
|
142
|
-
}
|
143
|
-
|
144
|
-
public String getErrorMessage()
|
145
|
-
{
|
146
|
-
if (validRecords == 0) {
|
147
|
-
return "No record processed";
|
148
|
-
}
|
149
|
-
if (errorRecords > 0) {
|
150
|
-
return String.format("%d invalid parts", errorParts);
|
151
|
-
}
|
152
|
-
if (errorRecords > 0) {
|
153
|
-
return String.format("%d invalid records", errorRecords);
|
154
|
-
}
|
155
|
-
return null;
|
156
|
-
}
|
157
|
-
}
|