@coveo/push-api-client 2.5.26 → 2.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/dist/definitions/help/fileConsumer.d.ts +5 -0
  2. package/dist/definitions/interfaces.d.ts +10 -0
  3. package/dist/docs/assets/search.js +1 -1
  4. package/dist/docs/classes/AnySecurityIdentityBuilder.html +5 -5
  5. package/dist/docs/classes/CatalogSource.html +17 -17
  6. package/dist/docs/classes/DocumentBuilder.html +29 -29
  7. package/dist/docs/classes/FieldAnalyser.html +13 -13
  8. package/dist/docs/classes/GroupSecurityIdentityBuilder.html +6 -6
  9. package/dist/docs/classes/PushSource.html +25 -25
  10. package/dist/docs/classes/UserSecurityIdentityBuilder.html +6 -6
  11. package/dist/docs/classes/VirtualGroupSecurityIdentityBuilder.html +6 -6
  12. package/dist/docs/classes/errors.FieldTypeInconsistencyError.html +4 -4
  13. package/dist/docs/classes/errors.InvalidDocument.html +4 -4
  14. package/dist/docs/classes/errors.InvalidPermanentId.html +4 -4
  15. package/dist/docs/classes/errors.NotAFileError.html +4 -4
  16. package/dist/docs/classes/errors.NotAJsonFileError.html +4 -4
  17. package/dist/docs/classes/errors.PrivilegeError.html +5 -5
  18. package/dist/docs/classes/errors.UnsupportedAttribute.html +4 -4
  19. package/dist/docs/classes/errors.UnsupportedFieldError.html +7 -7
  20. package/dist/docs/enums/PlatformEnvironment.html +6 -6
  21. package/dist/docs/enums/Region.html +1 -1
  22. package/dist/docs/enums/SourceVisibility.html +1 -1
  23. package/dist/docs/functions/parseAndGetDocumentBuilderFromJSONDocument.html +2 -2
  24. package/dist/docs/index.html +1 -1
  25. package/dist/docs/interfaces/BatchUpdateDocuments.html +4 -4
  26. package/dist/docs/interfaces/Document.html +15 -15
  27. package/dist/docs/interfaces/SecurityIdentity.html +5 -5
  28. package/dist/docs/interfaces/SecurityIdentityBuilder.html +3 -3
  29. package/dist/docs/interfaces/UploadBatchCallbackData.html +14 -5
  30. package/dist/docs/modules/errors.html +2 -2
  31. package/dist/docs/modules.html +1 -1
  32. package/dist/docs/types/BatchUpdateDocumentsFromFiles.html +2 -2
  33. package/dist/docs/types/CompressionType.html +2 -2
  34. package/dist/docs/types/FailedUploadCallback.html +2 -2
  35. package/dist/docs/types/Metadata.html +2 -2
  36. package/dist/docs/types/MetadataValue.html +2 -2
  37. package/dist/docs/types/PlatformUrlOptions.html +2 -2
  38. package/dist/docs/types/SecurityIdentityType.html +2 -2
  39. package/dist/docs/types/SuccessfulUploadCallback.html +2 -2
  40. package/dist/docs/types/Transformer.html +2 -2
  41. package/dist/docs/variables/BuiltInTransformers.html +2 -2
  42. package/dist/docs/variables/Source.html +2 -2
  43. package/dist/help/fileConsumer.js +25 -0
  44. package/dist/help/fileConsumer.js.map +1 -1
  45. package/dist/help/fileConsumer.spec.js +95 -15
  46. package/dist/help/fileConsumer.spec.js.map +1 -1
  47. package/dist/source/batchUploadDocumentsFromFile.js +9 -4
  48. package/dist/source/batchUploadDocumentsFromFile.js.map +1 -1
  49. package/package.json +6 -6
@@ -6,7 +6,7 @@
6
6
  <div id="tsd-toolbar-links"></div></div>
7
7
  <ul class="results">
8
8
  <li class="state loading">Preparing search index...</li>
9
- <li class="state failure">The search index is not available</li></ul><a href="" class="title">@coveo/push-api-client</a></div>
9
+ <li class="state failure">The search index is not available</li></ul><a href="../index.html" class="title">@coveo/push-api-client</a></div>
10
10
  <div class="table-cell" id="tsd-widgets"><a href="#" class="tsd-widget tsd-toolbar-icon menu no-caption" data-toggle="menu" aria-label="Menu"><svg width="16" height="16" viewBox="0 0 16 16" fill="none"><rect x="1" y="3" width="14" height="2" fill="var(--color-text)"></rect><rect x="1" y="7" width="14" height="2" fill="var(--color-text)"></rect><rect x="1" y="11" width="14" height="2" fill="var(--color-text)"></rect></svg></a></div></div></header>
11
11
  <div class="container container-main">
12
12
  <div class="col-8 col-content">
@@ -30,7 +30,7 @@
30
30
  <h5>str: <span class="tsd-signature-type">string</span></h5></li></ul></div>
31
31
  <h4 class="tsd-returns-title">Returns <span class="tsd-signature-type">string</span></h4></li></ul></li></ul></div><aside class="tsd-sources">
32
32
  <ul>
33
- <li>Defined in <a href="https://github.com/coveo/push-api-client.js/blob/3bde5ff/src/validation/transformers/transformer.ts#L1">src/validation/transformers/transformer.ts:1</a></li></ul></aside></div>
33
+ <li>Defined in <a href="https://github.com/coveo/push-api-client.js/blob/e7bae65/src/validation/transformers/transformer.ts#L1">src/validation/transformers/transformer.ts:1</a></li></ul></aside></div>
34
34
  <div class="col-4 col-menu menu-sticky-wrap menu-highlight">
35
35
  <div class="tsd-navigation settings">
36
36
  <details class="tsd-index-accordion"><summary class="tsd-accordion-summary">
@@ -6,7 +6,7 @@
6
6
  <div id="tsd-toolbar-links"></div></div>
7
7
  <ul class="results">
8
8
  <li class="state loading">Preparing search index...</li>
9
- <li class="state failure">The search index is not available</li></ul><a href="" class="title">@coveo/push-api-client</a></div>
9
+ <li class="state failure">The search index is not available</li></ul><a href="../index.html" class="title">@coveo/push-api-client</a></div>
10
10
  <div class="table-cell" id="tsd-widgets"><a href="#" class="tsd-widget tsd-toolbar-icon menu no-caption" data-toggle="menu" aria-label="Menu"><svg width="16" height="16" viewBox="0 0 16 16" fill="none"><rect x="1" y="3" width="14" height="2" fill="var(--color-text)"></rect><rect x="1" y="7" width="14" height="2" fill="var(--color-text)"></rect><rect x="1" y="11" width="14" height="2" fill="var(--color-text)"></rect></svg></a></div></div></header>
11
11
  <div class="container container-main">
12
12
  <div class="col-8 col-content">
@@ -17,7 +17,7 @@
17
17
  <h1>Variable BuiltInTransformers<code class="tsd-tag ts-flagConst">Const</code> </h1></div>
18
18
  <div class="tsd-signature">Built<wbr/>In<wbr/>Transformers<span class="tsd-signature-symbol">:</span> <span class="tsd-signature-type">Readonly</span><span class="tsd-signature-symbol">&lt;</span><span class="tsd-signature-symbol">{ </span><br/><span>    </span>identity<span class="tsd-signature-symbol">: </span><a href="../types/Transformer.html" class="tsd-signature-type" data-tsd-kind="Type alias">Transformer</a><span class="tsd-signature-symbol">; </span><br/><span>    </span>toLowerCase<span class="tsd-signature-symbol">: </span><a href="../types/Transformer.html" class="tsd-signature-type" data-tsd-kind="Type alias">Transformer</a><span class="tsd-signature-symbol">; </span><br/><span>    </span>toSnakeCase<span class="tsd-signature-symbol">: </span><a href="../types/Transformer.html" class="tsd-signature-type" data-tsd-kind="Type alias">Transformer</a><span class="tsd-signature-symbol">; </span><br/><span class="tsd-signature-symbol">}</span><span class="tsd-signature-symbol">&gt;</span><span class="tsd-signature-symbol"> = ...</span></div><aside class="tsd-sources">
19
19
  <ul>
20
- <li>Defined in <a href="https://github.com/coveo/push-api-client.js/blob/3bde5ff/src/validation/transformers/transformer.ts#L56">src/validation/transformers/transformer.ts:56</a></li></ul></aside></div>
20
+ <li>Defined in <a href="https://github.com/coveo/push-api-client.js/blob/e7bae65/src/validation/transformers/transformer.ts#L56">src/validation/transformers/transformer.ts:56</a></li></ul></aside></div>
21
21
  <div class="col-4 col-menu menu-sticky-wrap menu-highlight">
22
22
  <div class="tsd-navigation settings">
23
23
  <details class="tsd-index-accordion"><summary class="tsd-accordion-summary">
@@ -6,7 +6,7 @@
6
6
  <div id="tsd-toolbar-links"></div></div>
7
7
  <ul class="results">
8
8
  <li class="state loading">Preparing search index...</li>
9
- <li class="state failure">The search index is not available</li></ul><a href="" class="title">@coveo/push-api-client</a></div>
9
+ <li class="state failure">The search index is not available</li></ul><a href="../index.html" class="title">@coveo/push-api-client</a></div>
10
10
  <div class="table-cell" id="tsd-widgets"><a href="#" class="tsd-widget tsd-toolbar-icon menu no-caption" data-toggle="menu" aria-label="Menu"><svg width="16" height="16" viewBox="0 0 16 16" fill="none"><rect x="1" y="3" width="14" height="2" fill="var(--color-text)"></rect><rect x="1" y="7" width="14" height="2" fill="var(--color-text)"></rect><rect x="1" y="11" width="14" height="2" fill="var(--color-text)"></rect></svg></a></div></div></header>
11
11
  <div class="container container-main">
12
12
  <div class="col-8 col-content">
@@ -20,7 +20,7 @@
20
20
  <h3>Deprecated</h3><p>This class has been replaced by <a href="../classes/PushSource.html">PushSource</a></p>
21
21
  </div><aside class="tsd-sources">
22
22
  <ul>
23
- <li>Defined in <a href="https://github.com/coveo/push-api-client.js/blob/3bde5ff/src/source/source.ts#L6">src/source/source.ts:6</a></li></ul></aside></div>
23
+ <li>Defined in <a href="https://github.com/coveo/push-api-client.js/blob/e7bae65/src/source/source.ts#L6">src/source/source.ts:6</a></li></ul></aside></div>
24
24
  <div class="col-4 col-menu menu-sticky-wrap menu-highlight">
25
25
  <div class="tsd-navigation settings">
26
26
  <details class="tsd-index-accordion"><summary class="tsd-accordion-summary">
@@ -28,6 +28,7 @@ exports.FileConsumer = void 0;
28
28
  const parseFile_1 = require("../validation/parseFile");
29
29
  const path_1 = require("path");
30
30
  const generator_1 = require("./generator");
31
+ const bueno_1 = require("@coveo/bueno");
31
32
  /**
32
33
  * Util class to help injesting documents through a list of files and uploading generated document batches.
33
34
  */
@@ -94,8 +95,19 @@ class FileConsumer {
94
95
  };
95
96
  return { chunksToUpload, close };
96
97
  }
98
+ set expectedDocumentCount(count) {
99
+ this._totalDocumentCount = count;
100
+ this._remainingDocumentCount = count;
101
+ }
102
+ getRemainingDocumentCount(batch) {
103
+ if (this._remainingDocumentCount === undefined) {
104
+ return;
105
+ }
106
+ return (this._remainingDocumentCount -= batch.length);
107
+ }
97
108
  async uploadBatch(batch, fileNames) {
98
109
  let res;
110
+ const progress = this.getProgress(batch);
99
111
  try {
100
112
  res = await this.upload({
101
113
  addOrUpdate: batch,
@@ -106,14 +118,27 @@ class FileConsumer {
106
118
  this.cbFail(error, {
107
119
  files: fileNames,
108
120
  batch,
121
+ progress,
109
122
  });
110
123
  }
111
124
  this.cbSuccess({
112
125
  files: fileNames,
113
126
  batch,
114
127
  res,
128
+ progress,
115
129
  });
116
130
  }
131
+ getProgress(batch) {
132
+ const remainingDocumentCount = this.getRemainingDocumentCount(batch);
133
+ if ((0, bueno_1.isUndefined)(remainingDocumentCount) ||
134
+ (0, bueno_1.isUndefined)(this._totalDocumentCount)) {
135
+ return;
136
+ }
137
+ return {
138
+ remainingDocumentCount,
139
+ totalDocumentCount: this._totalDocumentCount,
140
+ };
141
+ }
117
142
  get accumulator() {
118
143
  return {
119
144
  size: 0,
@@ -1 +1 @@
1
- {"version":3,"file":"fileConsumer.js","sourceRoot":"","sources":["../../src/help/fileConsumer.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAOA,uDAAmF;AACnF,+BAA8B;AAC9B,2CAA6C;AAS7C;;GAEG;AACH,MAAa,YAAY;IAKvB;;;;OAIG;IACH,YACU,MAA+D,EAC/D,gBAAgD;QADhD,WAAM,GAAN,MAAM,CAAyD;QAC/D,qBAAgB,GAAhB,gBAAgB,CAAgC;QAVlD,cAAS,GAA6B,GAAG,EAAE,GAAE,CAAC,CAAC;QAC/C,WAAM,GAAyB,GAAG,EAAE,GAAE,CAAC,CAAC;IAU7C,CAAC;IAEG,KAAK,CAAC,OAAO,CAAC,KAAe,EAAE,OAA8B;QAClE,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAA,eAAQ,EAAC,IAAI,CAAC,CAAC,CAAC;QACtD,MAAM,EAAC,cAAc,EAAE,KAAK,EAAC,GAAG,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;QAEtE,MAAM,mBAAmB,GAAG,QAAQ,CAAC,EAAE,WAA8B;YACnE,KAAK,MAAM,MAAM,IAAI,cAAc,CAAC,WAAW,CAAC,EAAE;gBAChD,MAAM,MAAM,EAAE,CAAC;aAChB;QACH,CAAC,CAAC;QAEF,4CAA4C;QAC5C,MAAM,aAAa,GAAG;;gBACpB,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE;oBACrC,MAAM,WAAW,GAAG,cAAM,IAAA,sDAA0C,EAClE,QAAQ,EACR,OAAO,CACR,CAAA,CAAC;oBACF,cAAA,KAAK,CAAC,CAAC,iBAAA,cAAA,mBAAmB,CAAC,WAAW,CAAC,CAAA,CAAA,CAAA,CAAC;iBACzC;YACH,CAAC;SAAA,CAAC;QAEF,MAAM,IAAA,4BAAgB,EACpB,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EACxB,IAAI,CAAC,gBAAgB,CAAC,aAAa,CACpC,CAAC;QACF,MAAM,KAAK,EAAE,CAAC;IAChB,CAAC;IAEM,SAAS,CAAC,QAAkC;QACjD,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAEM,OAAO,CAAC,QAA8B;QAC3C,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC;IACzB,CAAC;IAEO,qBAAqB,CAC3B,SAAmB,EACnB,WAAW,GAAG,IAAI,CAAC,WAAW;QAE9B,MAAM,cAAc,GAAG,CAAC,gBAAmC,EAAE,EAAE;YAC7D,MAAM,eAAe,GAA+B,EAAE,CAAC;YAEvD,KAAK,MAAM,UAAU,IAAI,gBAAgB,EAAE;gBACzC,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CACjC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CACrC,CAAC;gBAEF,IAAI,WAAW,CAAC,IAAI,GAAG,SAAS,IAAI,YAAY,CAAC,gBAAgB,EAAE;oBACjE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC;oBAClC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;wBACrB,eAAe,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;qBACjE;oBACD,WAAW,CAAC,MAAM,GAAG,CAAC,UAAU,CAAC,CAAC;oBAClC,WAAW,CAAC,IAAI,GAAG,SAAS,CAAC;iBAC9B;qBAAM;oBACL,WAAW,CAAC,IAAI,IAAI,SAAS,CAAC;oBAC9B,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;iBACrC;aACF;YACD,OAAO,eAAe,CAAC;QACzB,CAAC,CAAC;QACF,MAAM,KAAK,GAAG,KAAK,IAAI,EAAE;YACvB,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QACxD,CAAC,CAAC;QACF,OAAO,EAAC,cAAc,EAAE,KAAK,EAAC,CAAC;IACjC,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,KAAwB,EAAE,SAAmB;QACrE,IAAI,GAA8B,CAAC;QACnC,IAAI;YACF,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC;gBACtB,WAAW,EAAE,KAAK;gBAClB,MAAM,EAAE,EAAE;aACX,CAAC,CAAC;SACJ;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;gBACjB,KAAK,EAAE,SAAS;gBAChB,KAAK;aACN,CAAC,CAAC;SACJ;QAED,IAAI,CAAC,SAAS,CAAC;YACb,KAAK,EAAE,SAAS;YAChB,KAAK;YACL,GAAG;SACJ,CAAC,CAAC;IACL,CAAC;IAED,IAAY,WAAW;QACrB,OAAO;YACL,IAAI,EAAE,CAAC;YACP,MAAM,EAAE,EAAE;SACX,CAAC;IACJ,CAAC;;AA7GH,oCA8GC;AA7GgB,6BAAgB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC"}
1
+ {"version":3,"file":"fileConsumer.js","sourceRoot":"","sources":["../../src/help/fileConsumer.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAQA,uDAAmF;AACnF,+BAA8B;AAC9B,2CAA6C;AAE7C,wCAAyC;AAQzC;;GAEG;AACH,MAAa,YAAY;IAOvB;;;;OAIG;IACH,YACU,MAA+D,EAC/D,gBAAgD;QADhD,WAAM,GAAN,MAAM,CAAyD;QAC/D,qBAAgB,GAAhB,gBAAgB,CAAgC;QAVlD,cAAS,GAA6B,GAAG,EAAE,GAAE,CAAC,CAAC;QAC/C,WAAM,GAAyB,GAAG,EAAE,GAAE,CAAC,CAAC;IAU7C,CAAC;IAEG,KAAK,CAAC,OAAO,CAAC,KAAe,EAAE,OAA8B;QAClE,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAA,eAAQ,EAAC,IAAI,CAAC,CAAC,CAAC;QACtD,MAAM,EAAC,cAAc,EAAE,KAAK,EAAC,GAAG,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;QAEtE,MAAM,mBAAmB,GAAG,QAAQ,CAAC,EAAE,WAA8B;YACnE,KAAK,MAAM,MAAM,IAAI,cAAc,CAAC,WAAW,CAAC,EAAE;gBAChD,MAAM,MAAM,EAAE,CAAC;aAChB;QACH,CAAC,CAAC;QAEF,4CAA4C;QAC5C,MAAM,aAAa,GAAG;;gBACpB,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE;oBACrC,MAAM,WAAW,GAAG,cAAM,IAAA,sDAA0C,EAClE,QAAQ,EACR,OAAO,CACR,CAAA,CAAC;oBACF,cAAA,KAAK,CAAC,CAAC,iBAAA,cAAA,mBAAmB,CAAC,WAAW,CAAC,CAAA,CAAA,CAAA,CAAC;iBACzC;YACH,CAAC;SAAA,CAAC;QAEF,MAAM,IAAA,4BAAgB,EACpB,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EACxB,IAAI,CAAC,gBAAgB,CAAC,aAAa,CACpC,CAAC;QACF,MAAM,KAAK,EAAE,CAAC;IAChB,CAAC;IAEM,SAAS,CAAC,QAAkC;QACjD,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAEM,OAAO,CAAC,QAA8B;QAC3C,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC;IACzB,CAAC;IAEO,qBAAqB,CAC3B,SAAmB,EACnB,WAAW,GAAG,IAAI,CAAC,WAAW;QAE9B,MAAM,cAAc,GAAG,CAAC,gBAAmC,EAAE,EAAE;YAC7D,MAAM,eAAe,GAA+B,EAAE,CAAC;YAEvD,KAAK,MAAM,UAAU,IAAI,gBAAgB,EAAE;gBACzC,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CACjC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CACrC,CAAC;gBAEF,IAAI,WAAW,CAAC,IAAI,GAAG,SAAS,IAAI,YAAY,CAAC,gBAAgB,EAAE;oBACjE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC;oBAClC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;wBACrB,eAAe,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;qBACjE;oBACD,WAAW,CAAC,MAAM,GAAG,CAAC,UAAU,CAAC,CAAC;oBAClC,WAAW,CAAC,IAAI,GAAG,SAAS,CAAC;iBAC9B;qBAAM;oBACL,WAAW,CAAC,IAAI,IAAI,SAAS,CAAC;oBAC9B,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;iBACrC;aACF;YACD,OAAO,eAAe,CAAC;QACzB,CAAC,CAAC;QACF,MAAM,KAAK,GAAG,KAAK,IAAI,EAAE;YACvB,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QACxD,CAAC,CAAC;QACF,OAAO,EAAC,cAAc,EAAE,KAAK,EAAC,CAAC;IACjC,CAAC;IAED,IAAW,qBAAqB,CAAC,KAAa;QAC5C,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;QACjC,IAAI,CAAC,uBAAuB,GAAG,KAAK,CAAC;IACvC,CAAC;IAEO,yBAAyB,CAAC,KAAwB;QACxD,IAAI,IAAI,CAAC,uBAAuB,KAAK,SAAS,EAAE;YAC9C,OAAO;SACR;QACD,OAAO,CAAC,IAAI,CAAC,uBAAuB,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC;IACxD,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,KAAwB,EAAE,SAAmB;QACrE,IAAI,GAA8B,CAAC;QACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI;YACF,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC;gBACtB,WAAW,EAAE,KAAK;gBAClB,MAAM,EAAE,EAAE;aACX,CAAC,CAAC;SACJ;QAAC,OAAO,KAAK,EAAE;YACd,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;gBACjB,KAAK,EAAE,SAAS;gBAChB,KAAK;gBACL,QAAQ;aACT,CAAC,CAAC;SACJ;QAED,IAAI,CAAC,SAAS,CAAC;YACb,KAAK,EAAE,SAAS;YAChB,KAAK;YACL,GAAG;YACH,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;IAEO,WAAW,CAAC,KAAwB;QAC1C,MAAM,sBAAsB,GAAG,IAAI,CAAC,yBAAyB,CAAC,KAAK,CAAC,CAAC;QACrE,IACE,IAAA,mBAAW,EAAC,sBAAsB,CAAC;YACnC,IAAA,mBAAW,EAAC,IAAI,CAAC,mBAAmB,CAAC,EACrC;YACA,OAAO;SACR;QACD,OAAO;YACL,sBAAsB;YACtB,kBAAkB,EAAE,IAAI,CAAC,mBAAmB;SAC7C,CAAC;IACJ,CAAC;IAED,IAAY,WAAW;QACrB,OAAO;YACL,IAAI,EAAE,CAAC;YACP,MAAM,EAAE,EAAE;SACX,CAAC;IACJ,CAAC;;AA5IH,oCA6IC;AA5IgB,6BAAgB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC"}
@@ -1,25 +1,37 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  jest.mock('axios');
4
- const path_1 = require("path");
5
- const process_1 = require("process");
4
+ jest.mock('../validation/parseFile');
5
+ const __1 = require("..");
6
6
  const fileConsumer_1 = require("./fileConsumer");
7
+ const parseFile_1 = require("../validation/parseFile");
8
+ const mockedParse = jest.mocked(parseFile_1.parseAndGetDocumentBuilderFromJSONDocument);
9
+ // generating 5 documents of 2 mb each
10
+ mockedParse.mockResolvedValue(generateDocBuilderBatch(5, 2));
11
+ function generateDocBuilderBatch(documentCount, documentSize) {
12
+ let counter = 1;
13
+ const randomCharacter = '#';
14
+ const bytes = documentSize * 1024 * 1024;
15
+ const data = Buffer.alloc(bytes, randomCharacter).toString();
16
+ const docBuilderFactory = () => new __1.DocumentBuilder(`https://url.com/${counter++}`, 'title').withData(data);
17
+ return [...Array(documentCount)].map(docBuilderFactory);
18
+ }
7
19
  describe('FileConsumer', () => {
8
20
  let fileConsumer;
9
21
  const fakeUpload = jest.fn();
10
- const pathToStub = (0, path_1.join)((0, process_1.cwd)(), 'src', '__stub__');
11
- const entries = [
12
- (0, path_1.join)(pathToStub, 'jsondocuments', 'batman.json'),
13
- (0, path_1.join)(pathToStub, 'jsondocuments', 'fightclub.json'),
14
- ];
22
+ const singleEntry = ['foo.json'];
23
+ const entries = ['bar.json', 'baz.json'];
15
24
  describe('when upload is successful', () => {
16
25
  beforeEach(() => {
17
26
  fileConsumer = new fileConsumer_1.FileConsumer(fakeUpload, { maxConcurrent: 10 });
18
- fakeUpload.mockResolvedValueOnce({
27
+ fakeUpload.mockResolvedValue({
19
28
  status: 202,
20
29
  statusText: 'All good',
21
30
  });
22
31
  });
32
+ afterEach(() => {
33
+ fakeUpload.mockReset();
34
+ });
23
35
  it('should call the success callback', async () => {
24
36
  const mockedHandleSuccess = jest.fn();
25
37
  fileConsumer.onSuccess(mockedHandleSuccess);
@@ -33,19 +45,72 @@ describe('FileConsumer', () => {
33
45
  expect(mockedHandleError).not.toHaveBeenCalled();
34
46
  });
35
47
  it('should only push JSON files', async () => {
48
+ const expectedSequence = [
49
+ 'https://url.com/1',
50
+ 'https://url.com/2',
51
+ 'https://url.com/3',
52
+ 'https://url.com/4',
53
+ 'https://url.com/5',
54
+ ].values();
36
55
  const handleBatchUpload = (data) => {
37
- const expected = [
38
- 'https://www.themoviedb.org/movie/268',
39
- 'https://www.themoviedb.org/movie/999',
40
- 'https://www.themoviedb.org/movie/550',
41
- ];
42
56
  for (let i = 0; i < data.batch.length; i++) {
43
57
  const documentBuilder = data.batch[i];
44
- expect(documentBuilder.marshal().documentId).toEqual(expected[i]);
58
+ const { documentId } = documentBuilder.marshal();
59
+ expect(documentId).toEqual(expectedSequence.next().value);
45
60
  }
46
61
  };
47
62
  fileConsumer.onSuccess(handleBatchUpload);
48
- await fileConsumer.consume(entries);
63
+ await fileConsumer.consume(singleEntry);
64
+ });
65
+ describe.each([
66
+ {
67
+ // 1 file with 5 documents of 2MB each => 10MB to upload => 3 batches: [4MB] [4MB] [2MB]
68
+ title: 'when consume a single file',
69
+ sequence: [2, 2, 1],
70
+ entries: ['foo.json'],
71
+ },
72
+ {
73
+ // 2 files with 5 documents each weighing 2MB => 20MB to upload => 5 batches: [4MB] [4MB] [4MB] [4MB] [4MB]
74
+ title: 'when consume 2 files',
75
+ sequence: [2, 2, 2, 2, 2],
76
+ entries: ['foo.json', 'bar.json'],
77
+ },
78
+ ])('$title', ({ sequence, entries }) => {
79
+ it(`should create ${sequence.length} batches`, async () => {
80
+ const batchOrder = sequence.values();
81
+ const handleBatchUpload = (data) => expect(data.batch.length).toEqual(batchOrder.next().value);
82
+ fileConsumer.onSuccess(handleBatchUpload);
83
+ await fileConsumer.consume(entries);
84
+ });
85
+ it(`should call callback ${sequence.length} times`, async () => {
86
+ const mockedHandleSuccess = jest.fn();
87
+ fileConsumer.onSuccess(mockedHandleSuccess);
88
+ await fileConsumer.consume(entries);
89
+ expect(mockedHandleSuccess).toHaveBeenCalledTimes(sequence.length);
90
+ });
91
+ });
92
+ describe('when the upload progress is returned', () => {
93
+ const mockedHandleSuccess = jest.fn();
94
+ const documentCount = 5;
95
+ beforeEach(async () => {
96
+ fileConsumer.expectedDocumentCount = documentCount;
97
+ fileConsumer.onSuccess(mockedHandleSuccess);
98
+ await fileConsumer.consume(singleEntry);
99
+ });
100
+ it('should call callback on every batch upload', async () => {
101
+ expect(mockedHandleSuccess).toHaveBeenCalledTimes(3);
102
+ });
103
+ it('should return remaining and total documents at each call', async () => {
104
+ expect(mockedHandleSuccess).toHaveBeenNthCalledWith(1, expect.objectContaining({
105
+ progress: { remainingDocumentCount: 3, totalDocumentCount: 5 },
106
+ }));
107
+ expect(mockedHandleSuccess).toHaveBeenNthCalledWith(2, expect.objectContaining({
108
+ progress: { remainingDocumentCount: 1, totalDocumentCount: 5 },
109
+ }));
110
+ expect(mockedHandleSuccess).toHaveBeenNthCalledWith(3, expect.objectContaining({
111
+ progress: { remainingDocumentCount: 0, totalDocumentCount: 5 },
112
+ }));
113
+ });
49
114
  });
50
115
  });
51
116
  describe('when upload is in error', () => {
@@ -57,6 +122,21 @@ describe('FileConsumer', () => {
57
122
  statusText: 'BAD_REQUEST',
58
123
  });
59
124
  });
125
+ it('should return the number of remaining documents', async () => {
126
+ const mockedHandleError = jest.fn();
127
+ const documentCount = 5;
128
+ fileConsumer.expectedDocumentCount = documentCount;
129
+ fileConsumer.onError(mockedHandleError);
130
+ await fileConsumer.consume(singleEntry);
131
+ expect(mockedHandleError).toHaveBeenCalledTimes(1);
132
+ expect(mockedHandleError).toHaveBeenCalledWith({
133
+ status: 412,
134
+ statusText: 'BAD_REQUEST',
135
+ }, expect.objectContaining({
136
+ progress: { remainingDocumentCount: 3, totalDocumentCount: 5 },
137
+ }) // 2 out of 5 document processed... 3 remaiming
138
+ );
139
+ });
60
140
  it('should call the error callback', async () => {
61
141
  const mockedHandleError = jest.fn();
62
142
  fileConsumer.onError(mockedHandleError);
@@ -1 +1 @@
1
- {"version":3,"file":"fileConsumer.spec.js","sourceRoot":"","sources":["../../src/help/fileConsumer.spec.ts"],"names":[],"mappings":";;AAAA,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAEnB,+BAA0B;AAC1B,qCAA4B;AAM5B,iDAA4C;AAE5C,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC5B,IAAI,YAA0B,CAAC;IAC/B,MAAM,UAAU,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;IAC7B,MAAM,UAAU,GAAG,IAAA,WAAI,EAAC,IAAA,aAAG,GAAE,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG;QACd,IAAA,WAAI,EAAC,UAAU,EAAE,eAAe,EAAE,aAAa,CAAC;QAChD,IAAA,WAAI,EAAC,UAAU,EAAE,eAAe,EAAE,gBAAgB,CAAC;KACpD,CAAC;IAEF,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACzC,UAAU,CAAC,GAAG,EAAE;YACd,YAAY,GAAG,IAAI,2BAAY,CAAC,UAAU,EAAE,EAAC,aAAa,EAAE,EAAE,EAAC,CAAC,CAAC;YACjE,UAAU,CAAC,qBAAqB,CAAC;gBAC/B,MAAM,EAAE,GAAG;gBACX,UAAU,EAAE,UAAU;aACvB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;YAChD,MAAM,mBAAmB,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;YACtC,YAAY,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;YAC5C,MAAM,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACpC,MAAM,CAAC,mBAAmB,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,iBAAiB,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;YACpC,YAAY,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;YACxC,MAAM,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACpC,MAAM,CAAC,iBAAiB,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;YAC3C,MAAM,iBAAiB,GAA6B,CAClD,IAA6B,EAC7B,EAAE;gBACF,MAAM,QAAQ,GAAG;oBACf,sCAAsC;oBACtC,sCAAsC;oBACtC,sCAAsC;iBACvC,CAAC;gBACF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBAC1C,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;oBACtC,MAAM,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;iBACnE;YACH,CAAC,CAAC;YAEF,YAAY,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;YAC1C,MAAM,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACvC,IAAI,YAA0B,CAAC;QAE/B,UAAU,CAAC,GAAG,EAAE;YACd,YAAY,GAAG,IAAI,2BAAY,CAAC,UAAU,EAAE,EAAC,aAAa,EAAE,EAAE,EAAC,CAAC,CAAC;YACjE,UAAU,CAAC,qBAAqB,CAAC;gBAC/B,MAAM,EAAE,GAAG;gBACX,UAAU,EAAE,aAAa;aAC1B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,iBAAiB,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;YACpC,YAAY,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;YACxC,MAAM,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAEpC,MAAM,CAAC,iBAAiB,CAAC,CAAC,gBAAgB,EAAE,CAAC;QAC/C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,gBAAgB,GAAyB,CAAC,GAAY,EAAE,EAAE;gBAC9D,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,EAAC,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,aAAa,EAAC,CAAC,CAAC;YAChE,CAAC,CAAC;YAEF,YAAY,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;YACvC,MAAM,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
1
+ {"version":3,"file":"fileConsumer.spec.js","sourceRoot":"","sources":["../../src/help/fileConsumer.spec.ts"],"names":[],"mappings":";;AAAA,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AACnB,IAAI,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;AACrC,0BAKY;AACZ,iDAA4C;AAC5C,uDAAmF;AAEnF,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,sDAA0C,CAAC,CAAC;AAE5E,sCAAsC;AACtC,WAAW,CAAC,iBAAiB,CAAC,uBAAuB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAC7D,SAAS,uBAAuB,CAC9B,aAAqB,EACrB,YAAoB;IAEpB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,MAAM,eAAe,GAAG,GAAG,CAAC;IAC5B,MAAM,KAAK,GAAG,YAAY,GAAG,IAAI,GAAG,IAAI,CAAC;IACzC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,eAAe,CAAC,CAAC,QAAQ,EAAE,CAAC;IAC7D,MAAM,iBAAiB,GAAG,GAAG,EAAE,CAC7B,IAAI,mBAAe,CAAC,mBAAmB,OAAO,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAE9E,OAAO,CAAC,GAAG,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;AAC1D,CAAC;AAED,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC5B,IAAI,YAA0B,CAAC;IAC/B,MAAM,UAAU,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;IAC7B,MAAM,WAAW,GAAG,CAAC,UAAU,CAAC,CAAC;IACjC,MAAM,OAAO,GAAG,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IAEzC,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACzC,UAAU,CAAC,GAAG,EAAE;YACd,YAAY,GAAG,IAAI,2BAAY,CAAC,UAAU,EAAE,EAAC,aAAa,EAAE,EAAE,EAAC,CAAC,CAAC;YACjE,UAAU,CAAC,iBAAiB,CAAC;gBAC3B,MAAM,EAAE,GAAG;gBACX,UAAU,EAAE,UAAU;aACvB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,SAAS,CAAC,GAAG,EAAE;YACb,UAAU,CAAC,SAAS,EAAE,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;YAChD,MAAM,mBAAmB,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;YACtC,YAAY,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;YAC5C,MAAM,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACpC,MAAM,CAAC,mBAAmB,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,iBAAiB,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;YACpC,YAAY,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;YACxC,MAAM,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACpC,MAAM,CAAC,iBAAiB,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;YAC3C,MAAM,gBAAgB,GAAG;gBACvB,mBAAmB;gBACnB,mBAAmB;gBACnB,mBAAmB;gBACnB,mBAAmB;gBACnB,mBAAmB;aACpB,CAAC,MAAM,EAAE,CAAC;YACX,MAAM,iBAAiB,GAAG,CAAC,IAA6B,EAAE,EAAE;gBAC1D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBAC1C,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;oBACtC,MAAM,EAAC,UAAU,EAAC,GAAG,eAAe,CAAC,OAAO,EAAE,CAAC;oBAC/C,MAAM,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC;iBAC3D;YACH,CAAC,CAAC;YAEF,YAAY,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;YAC1C,MAAM,YAAY,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,IAAI,CAAC;YACZ;gBACE,wFAAwF;gBACxF,KAAK,EAAE,4BAA4B;gBACnC,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;gBACnB,OAAO,EAAE,CAAC,UAAU,CAAC;aACtB;YACD;gBACE,2GAA2G;gBAC3G,KAAK,EAAE,sBAAsB;gBAC7B,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;gBACzB,OAAO,EAAE,CAAC,UAAU,EAAE,UAAU,CAAC;aAClC;SACF,CAAC,CAAC,QAAQ,EAAE,CAAC,EAAC,QAAQ,EAAE,OAAO,EAAC,EAAE,EAAE;YACnC,EAAE,CAAC,iBAAiB,QAAQ,CAAC,MAAM,UAAU,EAAE,KAAK,IAAI,EAAE;gBACxD,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACrC,MAAM,iBAAiB,GAA6B,CAAC,IAAI,EAAE,EAAE,CAC3D,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC;gBAE7D,YAAY,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;gBAC1C,MAAM,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACtC,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,wBAAwB,QAAQ,CAAC,MAAM,QAAQ,EAAE,KAAK,IAAI,EAAE;gBAC7D,MAAM,mBAAmB,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;gBACtC,YAAY,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;gBAC5C,MAAM,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;gBACpC,MAAM,CAAC,mBAAmB,CAAC,CAAC,qBAAqB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACrE,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,sCAAsC,EAAE,GAAG,EAAE;YACpD,MAAM,mBAAmB,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,aAAa,GAAG,CAAC,CAAC;YAExB,UAAU,CAAC,KAAK,IAAI,EAAE;gBACpB,YAAY,CAAC,qBAAqB,GAAG,aAAa,CAAC;gBACnD,YAAY,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;gBAC5C,MAAM,YAAY,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAC1C,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;gBAC1D,MAAM,CAAC,mBAAmB,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YACvD,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;gBACxE,MAAM,CAAC,mBAAmB,CAAC,CAAC,uBAAuB,CACjD,CAAC,EACD,MAAM,CAAC,gBAAgB,CAAC;oBACtB,QAAQ,EAAE,EAAC,sBAAsB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAC;iBAC7D,CAAC,CACH,CAAC;gBACF,MAAM,CAAC,mBAAmB,CAAC,CAAC,uBAAuB,CACjD,CAAC,EACD,MAAM,CAAC,gBAAgB,CAAC;oBACtB,QAAQ,EAAE,EAAC,sBAAsB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAC;iBAC7D,CAAC,CACH,CAAC;gBACF,MAAM,CAAC,mBAAmB,CAAC,CAAC,uBAAuB,CACjD,CAAC,EACD,MAAM,CAAC,gBAAgB,CAAC;oBACtB,QAAQ,EAAE,EAAC,sBAAsB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAC;iBAC7D,CAAC,CACH,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACvC,IAAI,YAA0B,CAAC;QAE/B,UAAU,CAAC,GAAG,EAAE;YACd,YAAY,GAAG,IAAI,2BAAY,CAAC,UAAU,EAAE,EAAC,aAAa,EAAE,EAAE,EAAC,CAAC,CAAC;YACjE,UAAU,CAAC,qBAAqB,CAAC;gBAC/B,MAAM,EAAE,GAAG;gBACX,UAAU,EAAE,aAAa;aAC1B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;YAC/D,MAAM,iBAAiB,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;YACpC,MAAM,aAAa,GAAG,CAAC,CAAC;YAExB,YAAY,CAAC,qBAAqB,GAAG,aAAa,CAAC;YACnD,YAAY,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;YACxC,MAAM,YAAY,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAExC,MAAM,CAAC,iBAAiB,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YACnD,MAAM,CAAC,iBAAiB,CAAC,CAAC,oBAAoB,CAC5C;gBACE,MAAM,EAAE,GAAG;gBACX,UAAU,EAAE,aAAa;aAC1B,EACD,MAAM,CAAC,gBAAgB,CAAC;gBACtB,QAAQ,EAAE,EAAC,sBAAsB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAC;aAC7D,CAAC,CAAC,+CAA+C;aACnD,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,iBAAiB,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC;YACpC,YAAY,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;YACxC,MAAM,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAEpC,MAAM,CAAC,iBAAiB,CAAC,CAAC,gBAAgB,EAAE,CAAC;QAC/C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,gBAAgB,GAAyB,CAAC,GAAY,EAAE,EAAE;gBAC9D,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,EAAC,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,aAAa,EAAC,CAAC,CAAC;YAChE,CAAC,CAAC;YAEF,YAAY,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;YACvC,MAAM,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
@@ -11,17 +11,22 @@ class BatchUploadDocumentsFromFilesReturn {
11
11
  this.consumer = new fileConsumer_1.FileConsumer((batch) => strategy.upload(batch), options);
12
12
  this.internalPromise = (async () => {
13
13
  var _a, _b;
14
+ let expectedDocumentCount = 0;
14
15
  const files = (0, file_1.getAllJsonFilesFromEntries)(filesOrDirectories);
15
16
  await ((_a = strategy.preUpload) === null || _a === void 0 ? void 0 : _a.call(strategy));
16
- if (options.createFields) {
17
- const analyser = new index_1.FieldAnalyser(platformClient);
18
- for (const filePath of files.values()) {
19
- const docBuilders = await (0, parseFile_1.parseAndGetDocumentBuilderFromJSONDocument)(filePath, options);
17
+ const analyser = new index_1.FieldAnalyser(platformClient);
18
+ for (const filePath of files.values()) {
19
+ const docBuilders = await (0, parseFile_1.parseAndGetDocumentBuilderFromJSONDocument)(filePath, options);
20
+ expectedDocumentCount += docBuilders.length;
21
+ if (options.createFields) {
20
22
  await analyser.add(docBuilders);
21
23
  }
24
+ }
25
+ if (options.createFields) {
22
26
  const report = analyser.report();
23
27
  await (0, fieldUtils_1.createFieldsFromReport)(platformClient, report);
24
28
  }
29
+ this.consumer.expectedDocumentCount = expectedDocumentCount;
25
30
  await this.consumer.consume(files, options);
26
31
  await ((_b = strategy.postUpload) === null || _b === void 0 ? void 0 : _b.call(strategy));
27
32
  }).bind(this);
@@ -1 +1 @@
1
- {"version":3,"file":"batchUploadDocumentsFromFile.js","sourceRoot":"","sources":["../../src/source/batchUploadDocumentsFromFile.ts"],"names":[],"mappings":";;;AACA,4DAAmE;AACnE,uCAAwD;AACxD,uDAAkD;AAClD,oCAMkB;AAElB,uDAAmF;AAEnF,MAAa,mCAAmC;IAI9C,YACE,cAA8B,EAC9B,QAAwB,EACxB,kBAA4B,EAC5B,OAAgD;QAEhD,IAAI,CAAC,QAAQ,GAAG,IAAI,2BAAY,CAC9B,CAAC,KAA2B,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,EACvD,OAAO,CACR,CAAC;QAEF,IAAI,CAAC,eAAe,GAAG,CAAC,KAAK,IAAI,EAAE;;YACjC,MAAM,KAAK,GAAG,IAAA,iCAA0B,EAAC,kBAAkB,CAAC,CAAC;YAC7D,MAAM,CAAA,MAAA,QAAQ,CAAC,SAAS,wDAAI,CAAA,CAAC;YAE7B,IAAI,OAAO,CAAC,YAAY,EAAE;gBACxB,MAAM,QAAQ,GAAG,IAAI,qBAAa,CAAC,cAAc,CAAC,CAAC;gBACnD,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE;oBACrC,MAAM,WAAW,GAAG,MAAM,IAAA,sDAA0C,EAClE,QAAQ,EACR,OAAO,CACR,CAAC;oBACF,MAAM,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;iBACjC;gBAED,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACjC,MAAM,IAAA,mCAAsB,EAAC,cAAc,EAAE,MAAM,CAAC,CAAC;aACtD;YAED,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAC5C,MAAM,CAAA,MAAA,QAAQ,CAAC,UAAU,wDAAI,CAAA,CAAC;QAChC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;IAEM,aAAa,CAAC,EAA4B;QAC/C,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,YAAY,CAAC,EAAwB;QAC1C,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,KAAK;QACV,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC;IAChC,CAAC;CACF;AAnDD,kFAmDC"}
1
+ {"version":3,"file":"batchUploadDocumentsFromFile.js","sourceRoot":"","sources":["../../src/source/batchUploadDocumentsFromFile.ts"],"names":[],"mappings":";;;AACA,4DAAmE;AACnE,uCAAwD;AACxD,uDAAkD;AAClD,oCAMkB;AAElB,uDAAmF;AAEnF,MAAa,mCAAmC;IAI9C,YACE,cAA8B,EAC9B,QAAwB,EACxB,kBAA4B,EAC5B,OAAgD;QAEhD,IAAI,CAAC,QAAQ,GAAG,IAAI,2BAAY,CAC9B,CAAC,KAA2B,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,EACvD,OAAO,CACR,CAAC;QAEF,IAAI,CAAC,eAAe,GAAG,CAAC,KAAK,IAAI,EAAE;;YACjC,IAAI,qBAAqB,GAAG,CAAC,CAAC;YAC9B,MAAM,KAAK,GAAG,IAAA,iCAA0B,EAAC,kBAAkB,CAAC,CAAC;YAC7D,MAAM,CAAA,MAAA,QAAQ,CAAC,SAAS,wDAAI,CAAA,CAAC;YAE7B,MAAM,QAAQ,GAAG,IAAI,qBAAa,CAAC,cAAc,CAAC,CAAC;YACnD,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE;gBACrC,MAAM,WAAW,GAAG,MAAM,IAAA,sDAA0C,EAClE,QAAQ,EACR,OAAO,CACR,CAAC;gBACF,qBAAqB,IAAI,WAAW,CAAC,MAAM,CAAC;gBAC5C,IAAI,OAAO,CAAC,YAAY,EAAE;oBACxB,MAAM,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;iBACjC;aACF;YAED,IAAI,OAAO,CAAC,YAAY,EAAE;gBACxB,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACjC,MAAM,IAAA,mCAAsB,EAAC,cAAc,EAAE,MAAM,CAAC,CAAC;aACtD;YAED,IAAI,CAAC,QAAQ,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;YAC5D,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAC5C,MAAM,CAAA,MAAA,QAAQ,CAAC,UAAU,wDAAI,CAAA,CAAC;QAChC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;IAEM,aAAa,CAAC,EAA4B;QAC/C,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,YAAY,CAAC,EAAwB;QAC1C,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,KAAK;QACV,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC;IAChC,CAAC;CACF;AAxDD,kFAwDC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@coveo/push-api-client",
3
- "version": "2.5.26",
3
+ "version": "2.6.1",
4
4
  "description": "Push API client",
5
5
  "main": "./dist/index.js",
6
6
  "files": [
@@ -19,23 +19,23 @@
19
19
  "devDependencies": {
20
20
  "@coveo/push-api-client": "file:.",
21
21
  "@types/jest": "29.2.0",
22
- "@types/node": "18.11.4",
23
- "@typescript-eslint/eslint-plugin": "5.40.1",
24
- "@typescript-eslint/parser": "5.40.1",
22
+ "@types/node": "18.11.8",
23
+ "@typescript-eslint/eslint-plugin": "5.41.0",
24
+ "@typescript-eslint/parser": "5.41.0",
25
25
  "cz-conventional-changelog": "3.3.0",
26
26
  "dotenv": "16.0.3",
27
27
  "eslint": "8.26.0",
28
28
  "eslint-config-prettier": "8.5.0",
29
29
  "gts": "3.1.1",
30
30
  "husky": "8.0.1",
31
- "jest": "29.2.1",
31
+ "jest": "29.2.2",
32
32
  "lint-staged": "13.0.3",
33
33
  "nodemon": "2.0.20",
34
34
  "prettier": "2.7.1",
35
35
  "standard-version": "9.5.0",
36
36
  "ts-jest": "29.0.3",
37
37
  "ts-node": "10.9.1",
38
- "typedoc": "0.23.18",
38
+ "typedoc": "0.23.19",
39
39
  "typescript": "4.8.4"
40
40
  },
41
41
  "husky": {