embulk-output-salesforce 0.1.2 → 0.1.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (8) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE.txt +21 -21
  3. data/build.gradle +75 -75
  4. data/gradle/wrapper/gradle-wrapper.jar +0 -0
  5. data/gradle/wrapper/gradle-wrapper.properties +2 -2
  6. data/gradlew +164 -164
  7. data/src/main/java/org/embulk/output/SalesforceOutputPlugin.java +533 -533
  8. metadata +13 -13
data/src/main/java/org/embulk/output/SalesforceOutputPlugin.java CHANGED
@@ -1,533 +1,533 @@
1
- package org.embulk.output;
2
-
3
- import com.google.common.base.Optional;
4
- import com.sforce.soap.partner.Connector;
5
- import com.sforce.soap.partner.DeleteResult;
6
- import com.sforce.soap.partner.DescribeSObjectResult;
7
- import com.sforce.soap.partner.Field;
8
- import com.sforce.soap.partner.FieldType;
9
- import com.sforce.soap.partner.GetUserInfoResult;
10
- import com.sforce.soap.partner.PartnerConnection;
11
- import com.sforce.soap.partner.SaveResult;
12
- import com.sforce.soap.partner.UpsertResult;
13
- import com.sforce.soap.partner.fault.ApiFault;
14
- import com.sforce.soap.partner.sobject.SObject;
15
- import com.sforce.ws.ConnectionException;
16
- import com.sforce.ws.ConnectorConfig;
17
- import java.io.File;
18
- import java.io.FileWriter;
19
- import java.io.IOException;
20
- import java.text.SimpleDateFormat;
21
- import java.util.ArrayList;
22
- import java.util.Calendar;
23
- import java.util.Date;
24
- import java.util.HashMap;
25
- import java.util.List;
26
- import java.util.Map;
27
- import org.embulk.config.CommitReport;
28
- import org.embulk.config.Config;
29
- import org.embulk.config.ConfigDefault;
30
- import org.embulk.config.ConfigDiff;
31
- import org.embulk.config.ConfigException;
32
- import org.embulk.config.ConfigSource;
33
- import org.embulk.config.Task;
34
- import org.embulk.config.TaskSource;
35
- import org.embulk.spi.Column;
36
- import org.embulk.spi.ColumnVisitor;
37
- import org.embulk.spi.Exec;
38
- import org.embulk.spi.OutputPlugin;
39
- import org.embulk.spi.Page;
40
- import org.embulk.spi.PageReader;
41
- import org.embulk.spi.Schema;
42
- import org.embulk.spi.TransactionalPageOutput;
43
- import org.joda.time.DateTime;
44
- import org.slf4j.Logger;
45
- import org.supercsv.cellprocessor.ift.CellProcessor;
46
- import org.supercsv.io.CsvListWriter;
47
- import org.supercsv.io.ICsvListWriter;
48
- import org.supercsv.prefs.CsvPreference;
49
-
50
- public class SalesforceOutputPlugin
51
- implements OutputPlugin
52
- {
53
- protected static Logger logger;
54
- private static PartnerConnection client = null;
55
- private static Map<String, String> externalIdToObjectNameMap = null;
56
-
57
- public interface PluginTask
58
- extends Task
59
- {
60
- @Config("username")
61
- public String getUsername();
62
-
63
- @Config("password")
64
- public String getPassword();
65
-
66
- @Config("login_endpoint")
67
- @ConfigDefault("\"https://login.salesforce.com\"")
68
- public Optional<String> getLoginEndpoint();
69
-
70
- @Config("sobject")
71
- public String getSObject();
72
-
73
- @Config("upsert_key")
74
- @ConfigDefault("null")
75
- public Optional<String> getUpsertKey();
76
-
77
- @Config("batch_size")
78
- @ConfigDefault("200")
79
- public Integer getBatchSize();
80
-
81
- @Config("action")
82
- @ConfigDefault("\"insert\"")
83
- public Optional<String> getAction();
84
-
85
- @Config("version")
86
- @ConfigDefault("34.0")
87
- public Optional<String> getVersion();
88
-
89
- @Config("result_dir")
90
- @ConfigDefault("null")
91
- public Optional<String> getResultDir();
92
- }
93
-
94
- @Override
95
- public ConfigDiff transaction(ConfigSource config,
96
- Schema schema, int taskCount,
97
- OutputPlugin.Control control)
98
- {
99
- PluginTask task = config.loadConfig(PluginTask.class);
100
- logger = Exec.getLogger(getClass());
101
-
102
- if (task.getResultDir().isPresent() && task.getResultDir().get() != null) {
103
- File resultDir = new File(task.getResultDir().get());
104
- if (!resultDir.exists() || !resultDir.isDirectory()) {
105
- logger.error("{} is not exist or is not directory.", task.getResultDir().get());
106
- throw new RuntimeException(task.getResultDir().get() + " is not exist or is not directory.");
107
- }
108
- }
109
-
110
- final String username = task.getUsername();
111
- final String password = task.getPassword();
112
- final String loginEndpoint = task.getLoginEndpoint().get();
113
- try {
114
- if (client == null) {
115
- ConnectorConfig connectorConfig = new ConnectorConfig();
116
- connectorConfig.setUsername(username);
117
- connectorConfig.setPassword(password);
118
- connectorConfig.setAuthEndpoint(loginEndpoint + "/services/Soap/u/" +task.getVersion().get() + "/");
119
-
120
- client = Connector.newConnection(connectorConfig);
121
- GetUserInfoResult userInfo = client.getUserInfo();
122
- logger.info("login successful with {}", userInfo.getUserName());
123
- externalIdToObjectNameMap = new HashMap<>();
124
- DescribeSObjectResult describeResult = client.describeSObject(task.getSObject());
125
- for (Field field : describeResult.getFields()) {
126
- if (field.getType() == FieldType.reference) {
127
- externalIdToObjectNameMap.put(field.getRelationshipName(), field.getReferenceTo()[0]);
128
- }
129
- }
130
- }
131
- } catch(ConnectionException ex) {
132
- logger.error("Login error. Please check your credentials.");
133
- throw new RuntimeException(ex);
134
- }
135
-
136
- control.run(task.dump());
137
- return Exec.newConfigDiff();
138
- }
139
-
140
- @Override
141
- public ConfigDiff resume(TaskSource taskSource,
142
- Schema schema, int taskCount,
143
- OutputPlugin.Control control)
144
- {
145
- throw new UnsupportedOperationException("salesforce output plugin does not support resuming");
146
- }
147
-
148
- @Override
149
- public void cleanup(TaskSource taskSource,
150
- Schema schema, int taskCount,
151
- List<CommitReport> successCommitReports)
152
- {
153
- logger.info("logout");
154
- try {
155
- if (client != null) {
156
- client.logout();
157
- }
158
- } catch (ConnectionException ex) {}
159
- }
160
-
161
- @Override
162
- public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex)
163
- {
164
- PluginTask task = taskSource.loadTask(PluginTask.class);
165
-
166
- PageReader reader = new PageReader(schema);
167
- return new SalesforcePageOutput(reader, client, task);
168
- }
169
-
170
- public class SalesforcePageOutput
171
- implements TransactionalPageOutput
172
- {
173
- private final String dateSuffix = new SimpleDateFormat("yyyyMMddhhmmssSSS").format(new Date());
174
-
175
- private final PageReader pageReader;
176
- private final PartnerConnection client;
177
- private final int batchSize;
178
- private List<SObject> records;
179
- private final String upsertKey;
180
- private final String sobject;
181
- private final String action;
182
- private final String resultDir;
183
- private Integer numOfSuccess = 0;
184
- private Integer numOfError = 0;
185
-
186
- public SalesforcePageOutput(final PageReader pageReader,
187
- PartnerConnection client, PluginTask task)
188
- {
189
- this.pageReader = pageReader;
190
- this.client = client;
191
- this.batchSize = task.getBatchSize();
192
- this.upsertKey = task.getUpsertKey().isPresent() ? task.getUpsertKey().get() : null;
193
- this.sobject = task.getSObject();
194
- this.action = task.getAction().isPresent() ? task.getAction().get() : null;
195
- this.resultDir = task.getResultDir().isPresent() ? task.getResultDir().get() : null;
196
- this.records = new ArrayList<>();
197
- }
198
-
199
- @Override
200
- public void add(Page page)
201
- {
202
- try {
203
- pageReader.setPage(page);
204
- while (pageReader.nextRecord()) {
205
- final SObject record = new SObject();
206
- record.setType(this.sobject);
207
-
208
- pageReader.getSchema().visitColumns(new ColumnVisitor() {
209
- @Override
210
- public void doubleColumn(Column column) {
211
- columnWithReferenceCheck(column.getName(), pageReader.getDouble(column));
212
- }
213
- @Override
214
- public void timestampColumn(Column column) {
215
- DateTime dt = new DateTime(pageReader.getTimestamp(column).getEpochSecond()*1000);
216
- Calendar cal = Calendar.getInstance();
217
- cal.clear();
218
- cal.setTimeZone(dt.getZone().toTimeZone());
219
- cal.set(dt.getYear(), dt.getMonthOfYear()-1, dt.getDayOfMonth(),
220
- dt.getHourOfDay(), dt.getMinuteOfHour(), dt.getSecondOfMinute());
221
- record.addField(column.getName(), cal);
222
- }
223
- @Override
224
- public void stringColumn(Column column) {
225
- columnWithReferenceCheck(column.getName(), pageReader.getString(column));
226
- }
227
- @Override
228
- public void longColumn(Column column) {
229
- columnWithReferenceCheck(column.getName(), pageReader.getLong(column));
230
- }
231
- @Override
232
- public void booleanColumn(Column column) {
233
- record.addField(column.getName(), pageReader.getBoolean(column));
234
- }
235
-
236
- private void columnWithReferenceCheck(String name, Object value) {
237
- if (name.indexOf('.') > 0) {
238
- String[] tokens = name.split("\\.");
239
- String referencesFieldName = tokens[0];
240
- String externalIdFieldName = tokens[1];
241
-
242
- SObject sObjRef = new SObject();
243
- String refFieldApiName = referencesFieldName.replaceAll("__R", "__r");
244
- if (externalIdToObjectNameMap.containsKey(refFieldApiName)) {
245
- sObjRef.setType(externalIdToObjectNameMap.get(refFieldApiName));
246
- } else {
247
- throw new ConfigException("Invalid Relationship Name '" + refFieldApiName + "'");
248
- }
249
- sObjRef.addField(externalIdFieldName, value);
250
- record.addField(referencesFieldName, sObjRef);
251
- } else {
252
- record.addField(name, value);
253
- }
254
- }
255
-
256
- });
257
- this.records.add(record);
258
-
259
- if (this.records.size() >= this.batchSize) {
260
- this.action(this.records);
261
- logger.info("Number of processed records: {}", this.numOfSuccess + this.numOfError);
262
- }
263
- }
264
-
265
- if (!this.records.isEmpty()) {
266
- this.action(this.records);
267
- logger.info("Number of processed records: {}", this.numOfSuccess + this.numOfError);
268
- }
269
- } catch (ConfigException ex) {
270
- logger.error("Configuration Error: {}", ex.getMessage());
271
- } catch (ApiFault ex) {
272
- logger.error("API Error: {}", ex.getExceptionMessage());
273
- } catch (ConnectionException ex) {
274
- logger.error("Connection Error: {}", ex.getMessage());
275
- } catch (IOException ex) {
276
- throw new RuntimeException(ex);
277
- }
278
- }
279
-
280
- @Override
281
- public void finish()
282
- {
283
-
284
- }
285
-
286
- @Override
287
- public void close()
288
- {
289
-
290
- }
291
-
292
- @Override
293
- public void abort()
294
- {
295
- }
296
-
297
- @Override
298
- public CommitReport commit()
299
- {
300
- return Exec.newCommitReport();
301
- }
302
-
303
- private void action(List<SObject> records) throws ConnectionException, IOException{
304
- switch(this.action){
305
- case "insert":
306
- SaveResult[] insertResults = client.create(
307
- (SObject[])this.records.toArray(new SObject[0]));
308
- countSaveResults(insertResults);
309
- if (resultDir != null) {
310
- ResultWrapper resultWrapper = new ResultWrapper(insertResults, null, null);
311
- createResultsFiles(records, resultWrapper);
312
- }
313
- break;
314
- case "upsert":
315
- UpsertResult[] upsertResults = client.upsert(
316
- this.upsertKey, (SObject[])this.records.toArray(new SObject[0]));
317
- countUpsertResults(upsertResults);
318
- if (resultDir != null) {
319
- ResultWrapper resultWrapper = new ResultWrapper(null, upsertResults, null);
320
- createResultsFiles(records, resultWrapper);
321
- }
322
- break;
323
- case "update":
324
- SaveResult[] updateResults = client.update(
325
- (SObject[])this.records.toArray(new SObject[0]));
326
- countSaveResults(updateResults);
327
- if (resultDir != null) {
328
- ResultWrapper resultWrapper = new ResultWrapper(updateResults, null, null);
329
- createResultsFiles(records, resultWrapper);
330
- }
331
- break;
332
- case "delete":
333
- List<String> ids = new ArrayList<>();
334
- for (SObject sobj : this.records) {
335
- ids.add(sobj.getId());
336
- }
337
- DeleteResult[] deleteResults = client.delete(ids.toArray(new String[0]));
338
- countDeleteResults(deleteResults);
339
- if (resultDir != null) {
340
- ResultWrapper resultWrapper = new ResultWrapper(null, null, deleteResults);
341
- createResultsFiles(records, resultWrapper);
342
- }
343
- break;
344
- }
345
- this.records = new ArrayList<>();
346
- }
347
-
348
- private void countSaveResults(SaveResult[] saveResults) {
349
- for (SaveResult saveResult : saveResults) {
350
- if (saveResult.isSuccess()) {
351
- this.numOfSuccess++;
352
- } else {
353
- this.numOfError++;
354
- }
355
- }
356
- }
357
-
358
- private void countUpsertResults(UpsertResult[] upsertResults) {
359
- for (UpsertResult upsertResult : upsertResults) {
360
- if (upsertResult.isSuccess()) {
361
- this.numOfSuccess++;
362
- } else {
363
- this.numOfError++;
364
- }
365
- }
366
- }
367
-
368
- private void countDeleteResults(DeleteResult[] deleteResults) {
369
- for (DeleteResult deleteResult : deleteResults) {
370
- if (deleteResult.isSuccess()) {
371
- this.numOfSuccess++;
372
- } else {
373
- this.numOfError++;
374
- }
375
- }
376
- }
377
-
378
- private List<String> createSuccessHeader() {
379
- final List<String> successHeader = new ArrayList<>();
380
- successHeader.add("Id");
381
- for (Column col : pageReader.getSchema().getColumns()) {
382
- successHeader.add(col.getName());
383
- }
384
- return successHeader;
385
- }
386
-
387
- private List<String> createErrorHeader() {
388
- final List<String> errorHeader = new ArrayList<>();
389
- for (Column col : pageReader.getSchema().getColumns()) {
390
- errorHeader.add(col.getName());
391
- }
392
- errorHeader.add("Error");
393
- return errorHeader;
394
- }
395
-
396
- private void createResultsFiles(List<SObject> records, ResultWrapper resultWrapper) throws IOException{
397
- ICsvListWriter successListWriter = null;
398
- ICsvListWriter errorListWriter = null;
399
- try {
400
- String successFileName = this.resultDir + "/success_" + dateSuffix + ".csv";
401
- Boolean isExistSuccessFile = new File(successFileName).exists();
402
- successListWriter = new CsvListWriter(new FileWriter(successFileName, true),
403
- CsvPreference.STANDARD_PREFERENCE);
404
- if (!isExistSuccessFile) {
405
- successListWriter.write(createSuccessHeader());
406
- }
407
-
408
- String errorFileName = this.resultDir + "/error_" + dateSuffix + ".csv";
409
- Boolean isExistErrorFile = new File(errorFileName).exists();
410
- errorListWriter = new CsvListWriter(new FileWriter(errorFileName, true),
411
- CsvPreference.STANDARD_PREFERENCE);
412
- if (!isExistErrorFile) {
413
- errorListWriter.write(createErrorHeader());
414
- }
415
-
416
- CellProcessor[] processors = new CellProcessor[pageReader.getSchema().getColumns().size() + 1];
417
- ArrayList<ArrayList<String>> errorValues = new ArrayList<>();
418
- for (Integer i = 0, imax = records.size(); i < imax; i++) {
419
- SObject record = records.get(i);
420
-
421
- List<String> values = new ArrayList<>();
422
- if (resultWrapper.getIsSuccess(i)) {
423
- values.add(resultWrapper.getId(i));
424
- }
425
- for (Column col : pageReader.getSchema().getColumns()) {
426
- Object obj = record.getSObjectField(col.getName());
427
- if (obj != null) {
428
- if (obj instanceof Calendar) {
429
- DateTime dt = new DateTime((Calendar)obj);
430
- values.add(dt.toString("yyyy-MM-dd'T'hh:mm:ssZZ"));
431
- } else {
432
- values.add(obj.toString());
433
- }
434
- } else {
435
- values.add("");
436
- }
437
- }
438
- if (!resultWrapper.getIsSuccess(i)) {
439
- StringBuilder sb = new StringBuilder();
440
- for (com.sforce.soap.partner.Error err : resultWrapper.getErrors(i)) {
441
- if (sb.length() > 0) {
442
- sb.append(";");
443
- }
444
- sb.append(err.getStatusCode())
445
- .append(":")
446
- .append(err.getMessage());
447
- }
448
-
449
- values.add(sb.toString());
450
- }
451
- if (resultWrapper.getIsSuccess(i)) {
452
- successListWriter.write(values);
453
- } else {
454
- errorListWriter.write(values);
455
- }
456
- }
457
- } finally {
458
- if(successListWriter != null ) {
459
- successListWriter.close();
460
- }
461
- if(errorListWriter != null ) {
462
- errorListWriter.close();
463
- }
464
- }
465
- }
466
- }
467
-
468
- public class ResultWrapper {
469
- private final SaveResult[] saveResult;
470
- private final UpsertResult[] upsertResult;
471
- private final DeleteResult[] deleteResult;
472
-
473
- public ResultWrapper(SaveResult[] saveResults,
474
- UpsertResult[] upsertResults, DeleteResult[] deleteResults) {
475
- this.saveResult = saveResults;
476
- this.upsertResult = upsertResults;
477
- this.deleteResult = deleteResults;
478
- }
479
-
480
- public Boolean isSaveResult() {
481
- return this.saveResult != null;
482
- }
483
-
484
- public Boolean isUpsertResult() {
485
- return this.upsertResult != null;
486
- }
487
-
488
- public Boolean isDeleteResult() {
489
- return this.deleteResult != null;
490
- }
491
-
492
- public String getId(Integer index) {
493
- if (this.isSaveResult()) {
494
- return this.saveResult[index].getId();
495
- } else if (this.isUpsertResult()) {
496
- return this.upsertResult[index].getId();
497
- } else if (this.isDeleteResult()) {
498
- return this.deleteResult[index].getId();
499
- }
500
- return null;
501
- }
502
-
503
- public Boolean getIsSuccess(Integer index) {
504
- if (this.isSaveResult()) {
505
- return this.saveResult[index].isSuccess();
506
- } else if (this.isUpsertResult()) {
507
- return this.upsertResult[index].isSuccess();
508
- } else if (this.isDeleteResult()) {
509
- return this.deleteResult[index].isSuccess();
510
- }
511
- return null;
512
- }
513
-
514
- public Boolean getIsCreated(Integer index) {
515
- if (this.isUpsertResult()) {
516
- return this.upsertResult[index].isCreated();
517
- }
518
- return null;
519
- }
520
-
521
- public com.sforce.soap.partner.Error[] getErrors(Integer index) {
522
- if (this.isSaveResult()) {
523
- return this.saveResult[index].getErrors();
524
- } else if (this.isUpsertResult()) {
525
- return this.upsertResult[index].getErrors();
526
- } else if (this.isDeleteResult()) {
527
- return this.deleteResult[index].getErrors();
528
- }
529
- return null;
530
- }
531
- }
532
-
533
- }
1
+ package org.embulk.output;
2
+
3
+ import com.google.common.base.Optional;
4
+ import com.sforce.soap.partner.Connector;
5
+ import com.sforce.soap.partner.DeleteResult;
6
+ import com.sforce.soap.partner.DescribeSObjectResult;
7
+ import com.sforce.soap.partner.Field;
8
+ import com.sforce.soap.partner.FieldType;
9
+ import com.sforce.soap.partner.GetUserInfoResult;
10
+ import com.sforce.soap.partner.PartnerConnection;
11
+ import com.sforce.soap.partner.SaveResult;
12
+ import com.sforce.soap.partner.UpsertResult;
13
+ import com.sforce.soap.partner.fault.ApiFault;
14
+ import com.sforce.soap.partner.sobject.SObject;
15
+ import com.sforce.ws.ConnectionException;
16
+ import com.sforce.ws.ConnectorConfig;
17
+ import java.io.File;
18
+ import java.io.FileWriter;
19
+ import java.io.IOException;
20
+ import java.text.SimpleDateFormat;
21
+ import java.util.ArrayList;
22
+ import java.util.Calendar;
23
+ import java.util.Date;
24
+ import java.util.HashMap;
25
+ import java.util.List;
26
+ import java.util.Map;
27
+ import org.embulk.config.TaskReport;
28
+ import org.embulk.config.Config;
29
+ import org.embulk.config.ConfigDefault;
30
+ import org.embulk.config.ConfigDiff;
31
+ import org.embulk.config.ConfigException;
32
+ import org.embulk.config.ConfigSource;
33
+ import org.embulk.config.Task;
34
+ import org.embulk.config.TaskSource;
35
+ import org.embulk.spi.Column;
36
+ import org.embulk.spi.ColumnVisitor;
37
+ import org.embulk.spi.Exec;
38
+ import org.embulk.spi.OutputPlugin;
39
+ import org.embulk.spi.Page;
40
+ import org.embulk.spi.PageReader;
41
+ import org.embulk.spi.Schema;
42
+ import org.embulk.spi.TransactionalPageOutput;
43
+ import org.joda.time.DateTime;
44
+ import org.slf4j.Logger;
45
+ import org.supercsv.cellprocessor.ift.CellProcessor;
46
+ import org.supercsv.io.CsvListWriter;
47
+ import org.supercsv.io.ICsvListWriter;
48
+ import org.supercsv.prefs.CsvPreference;
49
+
50
+ public class SalesforceOutputPlugin
51
+ implements OutputPlugin
52
+ {
53
+ protected static Logger logger;
54
+ private static PartnerConnection client = null;
55
+ private static Map<String, String> externalIdToObjectNameMap = null;
56
+
57
+ public interface PluginTask
58
+ extends Task
59
+ {
60
+ @Config("username")
61
+ public String getUsername();
62
+
63
+ @Config("password")
64
+ public String getPassword();
65
+
66
+ @Config("login_endpoint")
67
+ @ConfigDefault("\"https://login.salesforce.com\"")
68
+ public Optional<String> getLoginEndpoint();
69
+
70
+ @Config("sobject")
71
+ public String getSObject();
72
+
73
+ @Config("upsert_key")
74
+ @ConfigDefault("null")
75
+ public Optional<String> getUpsertKey();
76
+
77
+ @Config("batch_size")
78
+ @ConfigDefault("200")
79
+ public Integer getBatchSize();
80
+
81
+ @Config("action")
82
+ @ConfigDefault("\"insert\"")
83
+ public Optional<String> getAction();
84
+
85
+ @Config("version")
86
+ @ConfigDefault("34.0")
87
+ public Optional<String> getVersion();
88
+
89
+ @Config("result_dir")
90
+ @ConfigDefault("null")
91
+ public Optional<String> getResultDir();
92
+ }
93
+
94
+ @Override
95
+ public ConfigDiff transaction(ConfigSource config,
96
+ Schema schema, int taskCount,
97
+ OutputPlugin.Control control)
98
+ {
99
+ PluginTask task = config.loadConfig(PluginTask.class);
100
+ logger = Exec.getLogger(getClass());
101
+
102
+ if (task.getResultDir().isPresent() && task.getResultDir().get() != null) {
103
+ File resultDir = new File(task.getResultDir().get());
104
+ if (!resultDir.exists() || !resultDir.isDirectory()) {
105
+ logger.error("{} is not exist or is not directory.", task.getResultDir().get());
106
+ throw new RuntimeException(task.getResultDir().get() + " is not exist or is not directory.");
107
+ }
108
+ }
109
+
110
+ final String username = task.getUsername();
111
+ final String password = task.getPassword();
112
+ final String loginEndpoint = task.getLoginEndpoint().get();
113
+ try {
114
+ if (client == null) {
115
+ ConnectorConfig connectorConfig = new ConnectorConfig();
116
+ connectorConfig.setUsername(username);
117
+ connectorConfig.setPassword(password);
118
+ connectorConfig.setAuthEndpoint(loginEndpoint + "/services/Soap/u/" +task.getVersion().get() + "/");
119
+
120
+ client = Connector.newConnection(connectorConfig);
121
+ GetUserInfoResult userInfo = client.getUserInfo();
122
+ logger.info("login successful with {}", userInfo.getUserName());
123
+ externalIdToObjectNameMap = new HashMap<>();
124
+ DescribeSObjectResult describeResult = client.describeSObject(task.getSObject());
125
+ for (Field field : describeResult.getFields()) {
126
+ if (field.getType() == FieldType.reference) {
127
+ externalIdToObjectNameMap.put(field.getRelationshipName(), field.getReferenceTo()[0]);
128
+ }
129
+ }
130
+ }
131
+ } catch(ConnectionException ex) {
132
+ logger.error("Login error. Please check your credentials.");
133
+ throw new RuntimeException(ex);
134
+ }
135
+
136
+ control.run(task.dump());
137
+ return Exec.newConfigDiff();
138
+ }
139
+
140
+ @Override
141
+ public ConfigDiff resume(TaskSource taskSource,
142
+ Schema schema, int taskCount,
143
+ OutputPlugin.Control control)
144
+ {
145
+ throw new UnsupportedOperationException("salesforce output plugin does not support resuming");
146
+ }
147
+
148
+ @Override
149
+ public void cleanup(TaskSource taskSource,
150
+ Schema schema, int taskCount,
151
+ List<TaskReport> successTaskReports)
152
+ {
153
+ logger.info("logout");
154
+ try {
155
+ if (client != null) {
156
+ client.logout();
157
+ }
158
+ } catch (ConnectionException ex) {}
159
+ }
160
+
161
+ @Override
162
+ public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex)
163
+ {
164
+ PluginTask task = taskSource.loadTask(PluginTask.class);
165
+
166
+ PageReader reader = new PageReader(schema);
167
+ return new SalesforcePageOutput(reader, client, task);
168
+ }
169
+
170
+ public class SalesforcePageOutput
171
+ implements TransactionalPageOutput
172
+ {
173
+ private final String dateSuffix = new SimpleDateFormat("yyyyMMddhhmmssSSS").format(new Date());
174
+
175
+ private final PageReader pageReader;
176
+ private final PartnerConnection client;
177
+ private final int batchSize;
178
+ private List<SObject> records;
179
+ private final String upsertKey;
180
+ private final String sobject;
181
+ private final String action;
182
+ private final String resultDir;
183
+ private Integer numOfSuccess = 0;
184
+ private Integer numOfError = 0;
185
+
186
+ public SalesforcePageOutput(final PageReader pageReader,
187
+ PartnerConnection client, PluginTask task)
188
+ {
189
+ this.pageReader = pageReader;
190
+ this.client = client;
191
+ this.batchSize = task.getBatchSize();
192
+ this.upsertKey = task.getUpsertKey().isPresent() ? task.getUpsertKey().get() : null;
193
+ this.sobject = task.getSObject();
194
+ this.action = task.getAction().isPresent() ? task.getAction().get() : null;
195
+ this.resultDir = task.getResultDir().isPresent() ? task.getResultDir().get() : null;
196
+ this.records = new ArrayList<>();
197
+ }
198
+
199
+ @Override
200
+ public void add(Page page)
201
+ {
202
+ try {
203
+ pageReader.setPage(page);
204
+ while (pageReader.nextRecord()) {
205
+ final SObject record = new SObject();
206
+ record.setType(this.sobject);
207
+
208
+ pageReader.getSchema().visitColumns(new ColumnVisitor() {
209
+ @Override
210
+ public void doubleColumn(Column column) {
211
+ columnWithReferenceCheck(column.getName(), pageReader.getDouble(column));
212
+ }
213
+ @Override
214
+ public void timestampColumn(Column column) {
215
+ DateTime dt = new DateTime(pageReader.getTimestamp(column).getEpochSecond()*1000);
216
+ Calendar cal = Calendar.getInstance();
217
+ cal.clear();
218
+ cal.setTimeZone(dt.getZone().toTimeZone());
219
+ cal.set(dt.getYear(), dt.getMonthOfYear()-1, dt.getDayOfMonth(),
220
+ dt.getHourOfDay(), dt.getMinuteOfHour(), dt.getSecondOfMinute());
221
+ record.addField(column.getName(), cal);
222
+ }
223
+ @Override
224
+ public void stringColumn(Column column) {
225
+ columnWithReferenceCheck(column.getName(), pageReader.getString(column));
226
+ }
227
+ @Override
228
+ public void longColumn(Column column) {
229
+ columnWithReferenceCheck(column.getName(), pageReader.getLong(column));
230
+ }
231
+ @Override
232
+ public void booleanColumn(Column column) {
233
+ record.addField(column.getName(), pageReader.getBoolean(column));
234
+ }
235
+
236
+ private void columnWithReferenceCheck(String name, Object value) {
237
+ if (name.indexOf('.') > 0) {
238
+ String[] tokens = name.split("\\.");
239
+ String referencesFieldName = tokens[0];
240
+ String externalIdFieldName = tokens[1];
241
+
242
+ SObject sObjRef = new SObject();
243
+ String refFieldApiName = referencesFieldName.replaceAll("__R", "__r");
244
+ if (externalIdToObjectNameMap.containsKey(refFieldApiName)) {
245
+ sObjRef.setType(externalIdToObjectNameMap.get(refFieldApiName));
246
+ } else {
247
+ throw new ConfigException("Invalid Relationship Name '" + refFieldApiName + "'");
248
+ }
249
+ sObjRef.addField(externalIdFieldName, value);
250
+ record.addField(referencesFieldName, sObjRef);
251
+ } else {
252
+ record.addField(name, value);
253
+ }
254
+ }
255
+
256
+ });
257
+ this.records.add(record);
258
+
259
+ if (this.records.size() >= this.batchSize) {
260
+ this.action(this.records);
261
+ logger.info("Number of processed records: {}", this.numOfSuccess + this.numOfError);
262
+ }
263
+ }
264
+
265
+ if (!this.records.isEmpty()) {
266
+ this.action(this.records);
267
+ logger.info("Number of processed records: {}", this.numOfSuccess + this.numOfError);
268
+ }
269
+ } catch (ConfigException ex) {
270
+ logger.error("Configuration Error: {}", ex.getMessage());
271
+ } catch (ApiFault ex) {
272
+ logger.error("API Error: {}", ex.getExceptionMessage());
273
+ } catch (ConnectionException ex) {
274
+ logger.error("Connection Error: {}", ex.getMessage());
275
+ } catch (IOException ex) {
276
+ throw new RuntimeException(ex);
277
+ }
278
+ }
279
+
280
+ @Override
281
+ public void finish()
282
+ {
283
+
284
+ }
285
+
286
+ @Override
287
+ public void close()
288
+ {
289
+
290
+ }
291
+
292
+ @Override
293
+ public void abort()
294
+ {
295
+ }
296
+
297
+ @Override
298
+ public TaskReport commit()
299
+ {
300
+ return Exec.newTaskReport();
301
+ }
302
+
303
+ private void action(List<SObject> records) throws ConnectionException, IOException{
304
+ switch(this.action){
305
+ case "insert":
306
+ SaveResult[] insertResults = client.create(
307
+ (SObject[])this.records.toArray(new SObject[0]));
308
+ countSaveResults(insertResults);
309
+ if (resultDir != null) {
310
+ ResultWrapper resultWrapper = new ResultWrapper(insertResults, null, null);
311
+ createResultsFiles(records, resultWrapper);
312
+ }
313
+ break;
314
+ case "upsert":
315
+ UpsertResult[] upsertResults = client.upsert(
316
+ this.upsertKey, (SObject[])this.records.toArray(new SObject[0]));
317
+ countUpsertResults(upsertResults);
318
+ if (resultDir != null) {
319
+ ResultWrapper resultWrapper = new ResultWrapper(null, upsertResults, null);
320
+ createResultsFiles(records, resultWrapper);
321
+ }
322
+ break;
323
+ case "update":
324
+ SaveResult[] updateResults = client.update(
325
+ (SObject[])this.records.toArray(new SObject[0]));
326
+ countSaveResults(updateResults);
327
+ if (resultDir != null) {
328
+ ResultWrapper resultWrapper = new ResultWrapper(updateResults, null, null);
329
+ createResultsFiles(records, resultWrapper);
330
+ }
331
+ break;
332
+ case "delete":
333
+ List<String> ids = new ArrayList<>();
334
+ for (SObject sobj : this.records) {
335
+ ids.add(sobj.getId());
336
+ }
337
+ DeleteResult[] deleteResults = client.delete(ids.toArray(new String[0]));
338
+ countDeleteResults(deleteResults);
339
+ if (resultDir != null) {
340
+ ResultWrapper resultWrapper = new ResultWrapper(null, null, deleteResults);
341
+ createResultsFiles(records, resultWrapper);
342
+ }
343
+ break;
344
+ }
345
+ this.records = new ArrayList<>();
346
+ }
347
+
348
+ private void countSaveResults(SaveResult[] saveResults) {
349
+ for (SaveResult saveResult : saveResults) {
350
+ if (saveResult.isSuccess()) {
351
+ this.numOfSuccess++;
352
+ } else {
353
+ this.numOfError++;
354
+ }
355
+ }
356
+ }
357
+
358
+ private void countUpsertResults(UpsertResult[] upsertResults) {
359
+ for (UpsertResult upsertResult : upsertResults) {
360
+ if (upsertResult.isSuccess()) {
361
+ this.numOfSuccess++;
362
+ } else {
363
+ this.numOfError++;
364
+ }
365
+ }
366
+ }
367
+
368
+ private void countDeleteResults(DeleteResult[] deleteResults) {
369
+ for (DeleteResult deleteResult : deleteResults) {
370
+ if (deleteResult.isSuccess()) {
371
+ this.numOfSuccess++;
372
+ } else {
373
+ this.numOfError++;
374
+ }
375
+ }
376
+ }
377
+
378
+ private List<String> createSuccessHeader() {
379
+ final List<String> successHeader = new ArrayList<>();
380
+ successHeader.add("Id");
381
+ for (Column col : pageReader.getSchema().getColumns()) {
382
+ successHeader.add(col.getName());
383
+ }
384
+ return successHeader;
385
+ }
386
+
387
+ private List<String> createErrorHeader() {
388
+ final List<String> errorHeader = new ArrayList<>();
389
+ for (Column col : pageReader.getSchema().getColumns()) {
390
+ errorHeader.add(col.getName());
391
+ }
392
+ errorHeader.add("Error");
393
+ return errorHeader;
394
+ }
395
+
396
+ private void createResultsFiles(List<SObject> records, ResultWrapper resultWrapper) throws IOException{
397
+ ICsvListWriter successListWriter = null;
398
+ ICsvListWriter errorListWriter = null;
399
+ try {
400
+ String successFileName = this.resultDir + "/success_" + dateSuffix + ".csv";
401
+ Boolean isExistSuccessFile = new File(successFileName).exists();
402
+ successListWriter = new CsvListWriter(new FileWriter(successFileName, true),
403
+ CsvPreference.STANDARD_PREFERENCE);
404
+ if (!isExistSuccessFile) {
405
+ successListWriter.write(createSuccessHeader());
406
+ }
407
+
408
+ String errorFileName = this.resultDir + "/error_" + dateSuffix + ".csv";
409
+ Boolean isExistErrorFile = new File(errorFileName).exists();
410
+ errorListWriter = new CsvListWriter(new FileWriter(errorFileName, true),
411
+ CsvPreference.STANDARD_PREFERENCE);
412
+ if (!isExistErrorFile) {
413
+ errorListWriter.write(createErrorHeader());
414
+ }
415
+
416
+ CellProcessor[] processors = new CellProcessor[pageReader.getSchema().getColumns().size() + 1];
417
+ ArrayList<ArrayList<String>> errorValues = new ArrayList<>();
418
+ for (Integer i = 0, imax = records.size(); i < imax; i++) {
419
+ SObject record = records.get(i);
420
+
421
+ List<String> values = new ArrayList<>();
422
+ if (resultWrapper.getIsSuccess(i)) {
423
+ values.add(resultWrapper.getId(i));
424
+ }
425
+ for (Column col : pageReader.getSchema().getColumns()) {
426
+ Object obj = record.getSObjectField(col.getName());
427
+ if (obj != null) {
428
+ if (obj instanceof Calendar) {
429
+ DateTime dt = new DateTime((Calendar)obj);
430
+ values.add(dt.toString("yyyy-MM-dd'T'hh:mm:ssZZ"));
431
+ } else {
432
+ values.add(obj.toString());
433
+ }
434
+ } else {
435
+ values.add("");
436
+ }
437
+ }
438
+ if (!resultWrapper.getIsSuccess(i)) {
439
+ StringBuilder sb = new StringBuilder();
440
+ for (com.sforce.soap.partner.Error err : resultWrapper.getErrors(i)) {
441
+ if (sb.length() > 0) {
442
+ sb.append(";");
443
+ }
444
+ sb.append(err.getStatusCode())
445
+ .append(":")
446
+ .append(err.getMessage());
447
+ }
448
+
449
+ values.add(sb.toString());
450
+ }
451
+ if (resultWrapper.getIsSuccess(i)) {
452
+ successListWriter.write(values);
453
+ } else {
454
+ errorListWriter.write(values);
455
+ }
456
+ }
457
+ } finally {
458
+ if(successListWriter != null ) {
459
+ successListWriter.close();
460
+ }
461
+ if(errorListWriter != null ) {
462
+ errorListWriter.close();
463
+ }
464
+ }
465
+ }
466
+ }
467
+
468
+ public class ResultWrapper {
469
+ private final SaveResult[] saveResult;
470
+ private final UpsertResult[] upsertResult;
471
+ private final DeleteResult[] deleteResult;
472
+
473
+ public ResultWrapper(SaveResult[] saveResults,
474
+ UpsertResult[] upsertResults, DeleteResult[] deleteResults) {
475
+ this.saveResult = saveResults;
476
+ this.upsertResult = upsertResults;
477
+ this.deleteResult = deleteResults;
478
+ }
479
+
480
+ public Boolean isSaveResult() {
481
+ return this.saveResult != null;
482
+ }
483
+
484
+ public Boolean isUpsertResult() {
485
+ return this.upsertResult != null;
486
+ }
487
+
488
+ public Boolean isDeleteResult() {
489
+ return this.deleteResult != null;
490
+ }
491
+
492
+ public String getId(Integer index) {
493
+ if (this.isSaveResult()) {
494
+ return this.saveResult[index].getId();
495
+ } else if (this.isUpsertResult()) {
496
+ return this.upsertResult[index].getId();
497
+ } else if (this.isDeleteResult()) {
498
+ return this.deleteResult[index].getId();
499
+ }
500
+ return null;
501
+ }
502
+
503
+ public Boolean getIsSuccess(Integer index) {
504
+ if (this.isSaveResult()) {
505
+ return this.saveResult[index].isSuccess();
506
+ } else if (this.isUpsertResult()) {
507
+ return this.upsertResult[index].isSuccess();
508
+ } else if (this.isDeleteResult()) {
509
+ return this.deleteResult[index].isSuccess();
510
+ }
511
+ return null;
512
+ }
513
+
514
+ public Boolean getIsCreated(Integer index) {
515
+ if (this.isUpsertResult()) {
516
+ return this.upsertResult[index].isCreated();
517
+ }
518
+ return null;
519
+ }
520
+
521
+ public com.sforce.soap.partner.Error[] getErrors(Integer index) {
522
+ if (this.isSaveResult()) {
523
+ return this.saveResult[index].getErrors();
524
+ } else if (this.isUpsertResult()) {
525
+ return this.upsertResult[index].getErrors();
526
+ } else if (this.isDeleteResult()) {
527
+ return this.deleteResult[index].getErrors();
528
+ }
529
+ return null;
530
+ }
531
+ }
532
+
533
+ }