@devrev/airsync-sdk 2.0.0-beta.6 → 2.0.0-beta.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/README.md +8 -12
  2. package/dist/attachments-streaming/attachments-streaming-pool.d.ts.map +1 -1
  3. package/dist/attachments-streaming/attachments-streaming-pool.js +8 -9
  4. package/dist/common/constants.d.ts +0 -7
  5. package/dist/common/constants.d.ts.map +1 -1
  6. package/dist/common/constants.js +1 -28
  7. package/dist/common/helpers.d.ts.map +1 -1
  8. package/dist/common/helpers.js +1 -1
  9. package/dist/http/errors.d.ts +8 -0
  10. package/dist/http/errors.d.ts.map +1 -0
  11. package/dist/http/errors.js +31 -0
  12. package/dist/index.d.ts +18 -13
  13. package/dist/index.d.ts.map +1 -1
  14. package/dist/index.js +14 -11
  15. package/dist/logger/logger.d.ts.map +1 -1
  16. package/dist/logger/logger.interfaces.d.ts +2 -2
  17. package/dist/logger/logger.interfaces.d.ts.map +1 -1
  18. package/dist/mappers/mappers.d.ts +9 -3
  19. package/dist/mappers/mappers.d.ts.map +1 -1
  20. package/dist/mappers/mappers.interfaces.d.ts +0 -2
  21. package/dist/mappers/mappers.interfaces.d.ts.map +1 -1
  22. package/dist/mappers/mappers.js +33 -13
  23. package/dist/multithreading/adapters/base-adapter.d.ts +16 -35
  24. package/dist/multithreading/adapters/base-adapter.d.ts.map +1 -1
  25. package/dist/multithreading/adapters/base-adapter.js +89 -23
  26. package/dist/multithreading/adapters/extraction-adapter.d.ts +24 -20
  27. package/dist/multithreading/adapters/extraction-adapter.d.ts.map +1 -1
  28. package/dist/multithreading/adapters/extraction-adapter.js +175 -122
  29. package/dist/multithreading/adapters/loading-adapter.d.ts +13 -23
  30. package/dist/multithreading/adapters/loading-adapter.d.ts.map +1 -1
  31. package/dist/multithreading/adapters/loading-adapter.js +176 -301
  32. package/dist/multithreading/emit.d.ts.map +1 -1
  33. package/dist/multithreading/process-task.d.ts.map +1 -1
  34. package/dist/multithreading/process-task.js +24 -6
  35. package/dist/multithreading/spawn/spawn.d.ts +6 -1
  36. package/dist/multithreading/spawn/spawn.d.ts.map +1 -1
  37. package/dist/multithreading/spawn/spawn.helpers.d.ts.map +1 -1
  38. package/dist/multithreading/spawn/spawn.helpers.js +17 -52
  39. package/dist/multithreading/spawn/spawn.js +30 -26
  40. package/dist/multithreading/worker.js +15 -1
  41. package/dist/repo/repo.d.ts +4 -5
  42. package/dist/repo/repo.d.ts.map +1 -1
  43. package/dist/repo/repo.interfaces.d.ts +5 -4
  44. package/dist/repo/repo.interfaces.d.ts.map +1 -1
  45. package/dist/repo/repo.js +13 -15
  46. package/dist/state/base-state.d.ts +10 -5
  47. package/dist/state/base-state.d.ts.map +1 -1
  48. package/dist/state/base-state.js +39 -36
  49. package/dist/state/extraction-state.d.ts.map +1 -1
  50. package/dist/state/extraction-state.js +39 -4
  51. package/dist/state/state.interfaces.d.ts +13 -7
  52. package/dist/state/state.interfaces.d.ts.map +1 -1
  53. package/dist/state/state.interfaces.js +6 -8
  54. package/dist/state/time-value-resolver.d.ts.map +1 -1
  55. package/dist/state/time-value-resolver.js +3 -2
  56. package/dist/testing/mock-event.d.ts.map +1 -1
  57. package/dist/testing/mock-event.js +0 -8
  58. package/dist/testing/mock-server.d.ts +2 -0
  59. package/dist/testing/mock-server.d.ts.map +1 -1
  60. package/dist/testing/mock-server.interfaces.d.ts +6 -0
  61. package/dist/testing/mock-server.interfaces.d.ts.map +1 -1
  62. package/dist/testing/mock-server.js +37 -6
  63. package/dist/testing/test-adapters.d.ts +22 -0
  64. package/dist/testing/test-adapters.d.ts.map +1 -0
  65. package/dist/testing/test-adapters.js +32 -0
  66. package/dist/types/common.d.ts +4 -1
  67. package/dist/types/common.d.ts.map +1 -1
  68. package/dist/types/common.js +1 -1
  69. package/dist/types/errors.d.ts +13 -7
  70. package/dist/types/errors.d.ts.map +1 -1
  71. package/dist/types/errors.js +24 -0
  72. package/dist/types/extraction.d.ts +60 -44
  73. package/dist/types/extraction.d.ts.map +1 -1
  74. package/dist/types/extraction.js +10 -1
  75. package/dist/types/loading.d.ts +11 -30
  76. package/dist/types/loading.d.ts.map +1 -1
  77. package/dist/types/workers.d.ts +27 -23
  78. package/dist/types/workers.d.ts.map +1 -1
  79. package/dist/uploader/uploader.d.ts +1 -2
  80. package/dist/uploader/uploader.d.ts.map +1 -1
  81. package/dist/uploader/uploader.helpers.d.ts +3 -0
  82. package/dist/uploader/uploader.helpers.d.ts.map +1 -1
  83. package/dist/uploader/uploader.helpers.js +16 -0
  84. package/dist/uploader/uploader.interfaces.d.ts +4 -29
  85. package/dist/uploader/uploader.interfaces.d.ts.map +1 -1
  86. package/dist/uploader/uploader.js +4 -23
  87. package/docs/MIGRATION.md +651 -0
  88. package/package.json +24 -10
  89. package/MIGRATION.md +0 -1028
@@ -1 +1 @@
1
- {"version":3,"file":"base-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/base-adapter.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAC;AACxD,OAAO,EACL,YAAY,EACZ,SAAS,EACT,kBAAkB,EAClB,cAAc,EACf,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EACL,UAAU,EACV,oBAAoB,EAGrB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AAInD;;;GAGG;AACH,8BAAsB,WAAW,CAAC,cAAc;IAC9C,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,oBAAoB,CAAC;IACxC,gBAAgB,EAAE,OAAO,CAAC;IAE1B,OAAO,CAAC,UAAU,CAAkB;IACpC,OAAO,CAAC,oBAAoB,CAAc;IAC1C,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,CAElC;IAEH,SAAS,CAAC,YAAY,EAAE,SAAS,CAAC,cAAc,CAAC,CAAC;IAClD,SAAS,CAAC,QAAQ,EAAE,QAAQ,CAAC;gBAEjB,EACV,KAAK,EACL,YAAY,EACZ,OAAO,GACR,EAAE;QACD,KAAK,EAAE,YAAY,CAAC;QACpB,YAAY,EAAE,SAAS,CAAC,cAAc,CAAC,CAAC;QACxC,OAAO,CAAC,EAAE,oBAAoB,CAAC;KAChC;IAWD,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,IAAI,SAAS,CAAC,KAAK,EAAE,OAAO,EAK3B;IAED,qDAAqD;IACrD,IAAI,KAAK,IAAI,cAAc,CAE1B;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,cAAc,EAE9B;IAED,6DAA6D;IAC7D,IAAI,QAAQ,IAAI,QAAQ,CAEvB;IAED,IAAI,eAAe,kDAElB;IAEK,SAAS;IAIf,+EAA+E;IAC/E,SAAS,CAAC,QAAQ,CAAC,UAAU,CAC3B,YAAY,EAAE,kBAAkB,GAAG,eAAe,GACjD,OAAO,CAAC,IAAI,CAAC;IAEhB,kEAAkE;IAClE,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CACjC,YAAY,EAAE,kBAAkB,GAAG,eAAe,GACjD,SAAS;IAEZ;;;;OAIG;IACH,SAAS,CAAC,mBAAmB,CAE3B,YAAY,EAAE,kBAAkB,GAAG,eAAe,GACjD,cAAc,GAAG,SAAS;IAI7B,uEAAuE;IACvE,SAAS,CAAC,QAAQ,CAAC,SAAS,CAC1B,YAAY,EAAE,kBAAkB,GAAG,eAAe,GACjD,IAAI;IAEP;;;;OAIG;IACG,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;cAoBvC,IAAI,CAClB,YAAY,EAAE,kBAAkB,GAAG,eAAe,EAClD,IAAI,CAAC,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC;CAgEjB"}
1
+ {"version":3,"file":"base-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/base-adapter.ts"],"names":[],"mappings":"AASA,OAAO,EACL,YAAY,EAIb,MAAM,wBAAwB,CAAC;AAYhC;;;GAGG;AACH,8BAAsB,WAAW,CAAC,cAAc;IAC9C,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAI7B,OAAO,CAAC,iBAAiB,CAAkB;IAC3C,OAAO,CAAC,UAAU,CAAkB;IACpC,OAAO,CAAC,oBAAoB,CAAc;IAmC1C,2EAA2E;IAC3E,IAAI,SAAS,IAAI,OAAO,CAEvB;IAUD,qDAAqD;IACrD,IAAI,KAAK,IAAI,cAAc,CAE1B;IAED,IAAI,KAAK,CAAC,KAAK,EAAE,cAAc,EAE9B;IAOD,IAAI,eAAe,kDAElB;IAED;;;;OAIG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAIhC;;;;OAIG;IACG,cAAc,CAClB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EAAE,GACb,OAAO,CAAC;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC;IAY1B,kFAAkF;IAC5E,gBAAgB,CAAC,CAAC,GAAG,OAAO,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;YAmIvD,IAAI;CAmCnB"}
@@ -3,7 +3,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.BaseAdapter = void 0;
4
4
  const node_worker_threads_1 = require("node:worker_threads");
5
5
  const constants_1 = require("../../common/constants");
6
+ const errors_1 = require("../../http/errors");
6
7
  const logger_1 = require("../../logger/logger");
8
+ const errors_2 = require("../../types/errors");
7
9
  const workers_1 = require("../../types/workers");
8
10
  const uploader_1 = require("../../uploader/uploader");
9
11
  const emit_1 = require("../emit");
@@ -13,23 +15,31 @@ const spawn_helpers_1 = require("../spawn/spawn.helpers");
13
15
  * flow as a template method with mode-specific hooks.
14
16
  */
15
17
  class BaseAdapter {
18
+ /** @internal */
16
19
  constructor({ event, adapterState, options, }) {
20
+ this._hasWorkerEmitted = false;
17
21
  this._isTimeout = false;
22
+ /** @internal */
18
23
  this.timeoutSignal = new Promise((resolve) => {
19
24
  this.resolveTimeoutSignal = resolve;
20
25
  });
21
26
  this.event = event;
22
27
  this.options = options;
23
28
  this.adapterState = adapterState;
24
- this.hasWorkerEmitted = false;
25
29
  this.uploader = new uploader_1.Uploader({
26
30
  event,
27
31
  options,
28
32
  });
29
33
  }
34
+ /** @internal */
35
+ get hasWorkerEmitted() {
36
+ return this._hasWorkerEmitted;
37
+ }
38
+ /** Set once the soft timeout fires; the task should wrap up and return. */
30
39
  get isTimeout() {
31
40
  return this._isTimeout;
32
41
  }
42
+ /** @internal */
33
43
  set isTimeout(value) {
34
44
  this._isTimeout = value;
35
45
  if (value) {
@@ -43,17 +53,49 @@ class BaseAdapter {
43
53
  set state(value) {
44
54
  this.adapterState.state = value;
45
55
  }
46
- /** SDK-internal bookkeeping state; not for connector use. */
56
+ /** @internal */
47
57
  get sdkState() {
48
58
  return this.adapterState.sdkState;
49
59
  }
50
60
  get extractionScope() {
51
61
  return this.adapterState.extractionScope;
52
62
  }
63
+ /**
64
+ * Persists connector and SDK state right now. State is saved on every emit
65
+ * anyway; call this only to record intent before an external write that
66
+ * must not be repeated. Throws if the save fails.
67
+ */
53
68
  async postState() {
54
69
  await this.adapterState.postState();
55
70
  }
56
71
  /**
72
+ * Uploads connector-owned data (e.g. state too large to keep inline) as a
73
+ * gzipped JSONL artifact. Unlike repo uploads it is not reported in the
74
+ * emitted event. Throws on failure.
75
+ */
76
+ async uploadArtifact(itemType, data) {
77
+ var _a;
78
+ const { artifact, error } = await this.uploader.upload(itemType, data);
79
+ if (error || !artifact) {
80
+ throw new errors_1.AirSyncHttpError(`Failed to upload artifact of type ${itemType}. ${(_a = error === null || error === void 0 ? void 0 : error.message) !== null && _a !== void 0 ? _a : 'No artifact returned.'}`);
81
+ }
82
+ return { id: artifact.id };
83
+ }
84
+ /** Downloads an artifact written by {@link uploadArtifact}. Throws on failure. */
85
+ async downloadArtifact(artifactId) {
86
+ const { response, error } = await this.uploader.getJsonObjectByArtifactId({
87
+ artifactId,
88
+ isGzipped: true,
89
+ });
90
+ if (error !== undefined || response === undefined) {
91
+ throw error instanceof errors_1.AirSyncHttpError
92
+ ? error
93
+ : new errors_1.AirSyncHttpError(`Failed to download artifact ${artifactId}. ${(0, logger_1.serializeError)(error)}`);
94
+ }
95
+ return (Array.isArray(response) ? response : [response]);
96
+ }
97
+ /**
98
+ * @internal
57
99
  * Mode-specific worker metadata merged into the emitted event. The library
58
100
  * version and state-date range are added centrally in `emit`, so subclasses
59
101
  * only contribute their own statistics.
@@ -63,10 +105,14 @@ class BaseAdapter {
63
105
  newEventType) {
64
106
  return undefined;
65
107
  }
108
+ /** @internal Runs before a failure is reported via `emitError`. */
109
+ beforeErrorEmit() { }
66
110
  /**
67
111
  * Maps a {@link TaskResult} to the phase-appropriate platform event and emits
68
112
  * it exactly once. Invoked by the worker driver, not by connectors —
69
113
  * connectors signal outcomes by returning a `TaskResult`, never by emitting.
114
+ *
115
+ * @internal
70
116
  */
71
117
  async emitFromResult(result) {
72
118
  const { eventType, illegal } = (0, spawn_helpers_1.getEventTypeForResult)(this.event.payload.event_type, result.status);
@@ -84,37 +130,46 @@ class BaseAdapter {
84
130
  }
85
131
  await this.emit(eventType, data);
86
132
  }
133
+ /** @internal */
87
134
  async emit(newEventType, data) {
88
- var _a;
89
- if (this.hasWorkerEmitted) {
135
+ if (this._hasWorkerEmitted) {
90
136
  console.warn(`Trying to emit event with event type: ${newEventType}. Ignoring emit request because it has already been emitted.`);
91
137
  return;
92
138
  }
93
139
  try {
94
140
  await this.beforeEmit(newEventType);
141
+ // Save state on every emit, except for start and delete events
142
+ if (!constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
143
+ console.log(`Saving state before emitting event with event type: ${newEventType}.`);
144
+ await this.adapterState.postState();
145
+ }
95
146
  }
96
147
  catch (error) {
97
- console.error('Error while preparing to emit event', error);
98
- node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
99
- this.hasWorkerEmitted = true;
148
+ console.error('Error while preparing event for emission.', (0, logger_1.serializeError)(error));
149
+ await this.emitError(error);
100
150
  return;
101
151
  }
102
- // Save state on every emit, except for start and delete events
103
- if (!constants_1.STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) {
104
- console.log(`Saving state before emitting event with event type: ${newEventType}.`);
105
- try {
106
- await this.adapterState.postState();
107
- }
108
- catch (error) {
109
- console.error('Error while posting state', error);
110
- node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
111
- this.hasWorkerEmitted = true;
112
- return;
113
- }
152
+ await this.send(newEventType, data);
153
+ }
154
+ /**
155
+ * Emits the phase error for a failed task or pre-emit step. Skips the
156
+ * pre-emit flush and state save, so a failed batch isn't retried; artifacts
157
+ * uploaded before the failure are still reported.
158
+ *
159
+ * @internal
160
+ */
161
+ async emitError(error) {
162
+ if (this._hasWorkerEmitted) {
163
+ return;
114
164
  }
165
+ this.beforeErrorEmit();
166
+ const { eventType } = (0, spawn_helpers_1.getEventTypeForResult)(this.event.payload.event_type, 'error');
167
+ await this.send(eventType, { error: { message: (0, logger_1.serializeError)(error) } });
168
+ }
169
+ async send(newEventType, data) {
115
170
  try {
116
- if ((_a = data === null || data === void 0 ? void 0 : data.error) === null || _a === void 0 ? void 0 : _a.message) {
117
- data.error.message = (0, logger_1.truncateMessage)(data.error.message);
171
+ if (data === null || data === void 0 ? void 0 : data.error) {
172
+ data.error = toWireError(data.error);
118
173
  }
119
174
  await (0, emit_1.emit)({
120
175
  eventType: newEventType,
@@ -128,13 +183,24 @@ class BaseAdapter {
128
183
  };
129
184
  this.afterEmit(newEventType);
130
185
  node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(message);
131
- this.hasWorkerEmitted = true;
186
+ this._hasWorkerEmitted = true;
132
187
  }
133
188
  catch (error) {
134
189
  console.error(`Error while emitting event with event type: ${newEventType}.`, (0, logger_1.serializeError)(error));
135
190
  node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage(workers_1.WorkerMessageSubject.WorkerMessageExit);
136
- this.hasWorkerEmitted = true;
191
+ this._hasWorkerEmitted = true;
137
192
  }
138
193
  }
139
194
  }
140
195
  exports.BaseAdapter = BaseAdapter;
196
+ /**
197
+ * The platform matches error codes against the whole message, so a coded
198
+ * error is sent as just the code and its details only go to the logs.
199
+ */
200
+ function toWireError({ message, code }) {
201
+ if (!code) {
202
+ return { message: (0, logger_1.truncateMessage)(message) };
203
+ }
204
+ console.error(`Reporting ${code}: ${message}`);
205
+ return { message: (0, errors_2.formatErrorCode)(code) };
206
+ }
@@ -1,41 +1,45 @@
1
1
  import { Repo } from '../../repo/repo';
2
2
  import { NormalizedAttachment, RepoInterface } from '../../repo/repo.interfaces';
3
- import { BaseState } from '../../state/state';
4
- import { AirSyncEvent, EventData, ExternalSystemAttachmentProcessors, ExternalSystemAttachmentStreamingFunction, ExtractorEventType, ProcessAttachmentReturnType, WorkerMetadata } from '../../types/extraction';
5
- import { LoaderEventType } from '../../types/loading';
6
- import { TaskResult, WorkerAdapterOptions } from '../../types/workers';
3
+ import { ExternalSystemAttachmentProcessors, ExternalSystemAttachmentStreamingFunction, ItemTypeCount, ProcessAttachmentReturnType } from '../../types/extraction';
4
+ import { TaskResult } from '../../types/workers';
7
5
  import { Artifact } from '../../uploader/uploader.interfaces';
8
6
  import { BaseAdapter } from './base-adapter';
9
7
  /** Adapter passed to extraction tasks (repos, artifacts, attachment streaming). @public */
10
8
  export declare class ExtractionAdapter<ConnectorState> extends BaseAdapter<ConnectorState> {
11
9
  private _artifacts;
10
+ private _preExtractionItemCounts?;
12
11
  private repos;
13
12
  private lastExtractedItemType?;
14
13
  private currentEventDataLength;
15
- constructor(params: {
16
- event: AirSyncEvent;
17
- adapterState: BaseState<ConnectorState>;
18
- options?: WorkerAdapterOptions;
19
- });
20
14
  /** Defaults to true if the scope is empty or the item type is not listed. */
21
15
  shouldExtract(itemType: string): boolean;
22
16
  initializeRepos(repos: RepoInterface[]): void;
17
+ private createRepo;
18
+ /**
19
+ * SSOR attachments created outside the attachments phase are held in state
20
+ * and reported with ATTACHMENT_EXTRACTION_DONE instead of the current event.
21
+ */
22
+ private isDeferredRepo;
23
+ private isAttachmentsPhase;
24
+ private getOrCreateSsorRepo;
23
25
  getRepo(itemType: string): Repo | undefined;
24
- get artifacts(): Artifact[];
25
- set artifacts(artifacts: Artifact[]);
26
- protected beforeEmit(newEventType: ExtractorEventType | LoaderEventType): Promise<void>;
27
- protected buildEmitPayload(newEventType: ExtractorEventType | LoaderEventType): EventData;
26
+ /** Artifacts uploaded so far that the next emit will report. */
27
+ get artifacts(): readonly Artifact[];
28
+ private addArtifacts;
28
29
  /**
29
- * Attaches the last-extracted item type and its created/modified date ranges
30
- * to data/attachment done/progress events, mirroring the metadata v1 sent.
30
+ * Per-record-type counts for sync-duration estimation. Set during the
31
+ * metadata phase; the SDK attaches them to the metadata-done event.
31
32
  */
32
- protected buildWorkerMetadata(newEventType: ExtractorEventType | LoaderEventType): WorkerMetadata | undefined;
33
+ get preExtractionItemCounts(): ItemTypeCount[] | undefined;
34
+ set preExtractionItemCounts(counts: ItemTypeCount[] | undefined);
33
35
  private toRfc3339Timestamp;
34
- protected afterEmit(): void;
35
- uploadAllRepos(): Promise<void>;
36
+ private uploadAllRepos;
37
+ /**
38
+ * Streams one attachment into DevRev. Usable in any extraction phase; SSOR
39
+ * records created outside the attachments phase are reported with
40
+ * ATTACHMENT_EXTRACTION_DONE.
41
+ */
36
42
  processAttachment(attachment: NormalizedAttachment, stream: ExternalSystemAttachmentStreamingFunction): Promise<ProcessAttachmentReturnType>;
37
- /** Destroys a stream to prevent memory leaks. */
38
- private destroyHttpStream;
39
43
  streamAttachments<NewBatch>({ stream, processors, batchSize, }: {
40
44
  stream: ExternalSystemAttachmentStreamingFunction;
41
45
  processors?: ExternalSystemAttachmentProcessors<ConnectorState, NormalizedAttachment[], NewBatch>;
@@ -1 +1 @@
1
- {"version":3,"file":"extraction-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/extraction-adapter.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EACL,YAAY,EACZ,SAAS,EACT,kCAAkC,EAClC,yCAAyC,EACzC,kBAAkB,EAElB,2BAA2B,EAC3B,cAAc,EACf,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AACvE,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAE9E,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,2FAA2F;AAC3F,qBAAa,iBAAiB,CAC5B,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,qBAAqB,CAAC,CAAS;IACvC,OAAO,CAAC,sBAAsB,CAAa;gBAE/B,MAAM,EAAE;QAClB,KAAK,EAAE,YAAY,CAAC;QACpB,YAAY,EAAE,SAAS,CAAC,cAAc,CAAC,CAAC;QACxC,OAAO,CAAC,EAAE,oBAAoB,CAAC;KAChC;IAKD,6EAA6E;IAC7E,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAOxC,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IA0CtC,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAW3C,IAAI,SAAS,IAAI,QAAQ,EAAE,CAE1B;IAED,IAAI,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,EAIlC;cAEe,UAAU,CACxB,YAAY,EAAE,kBAAkB,GAAG,eAAe,GACjD,OAAO,CAAC,IAAI,CAAC;IAyChB,SAAS,CAAC,gBAAgB,CACxB,YAAY,EAAE,kBAAkB,GAAG,eAAe,GACjD,SAAS;IAOZ;;;OAGG;cACgB,mBAAmB,CACpC,YAAY,EAAE,kBAAkB,GAAG,eAAe,GACjD,cAAc,GAAG,SAAS;IAmC7B,OAAO,CAAC,kBAAkB;IAQ1B,SAAS,CAAC,SAAS,IAAI,IAAI;IAIrB,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAU/B,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IA2GvC,iDAAiD;IACjD,OAAO,CAAC,iBAAiB;IAcnB,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,UAAU,CAAC;CA8HxB"}
1
+ {"version":3,"file":"extraction-adapter.d.ts","sourceRoot":"","sources":["../../../src/multithreading/adapters/extraction-adapter.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EACL,oBAAoB,EACpB,aAAa,EACd,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAIL,kCAAkC,EAClC,yCAAyC,EAEzC,aAAa,EACb,2BAA2B,EAE5B,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAkB,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjE,OAAO,EAAE,QAAQ,EAAkB,MAAM,oCAAoC,CAAC;AAE9E,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,2FAA2F;AAC3F,qBAAa,iBAAiB,CAC5B,cAAc,CACd,SAAQ,WAAW,CAAC,cAAc,CAAC;IACnC,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,wBAAwB,CAAC,CAAkB;IACnD,OAAO,CAAC,KAAK,CAAc;IAC3B,OAAO,CAAC,qBAAqB,CAAC,CAAS;IACvC,OAAO,CAAC,sBAAsB,CAAa;IAY3C,6EAA6E;IAC7E,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAOxC,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE;IAatC,OAAO,CAAC,UAAU;IAoDlB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,kBAAkB;IAQ1B,OAAO,CAAC,mBAAmB;IAS3B,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS;IAW3C,gEAAgE;IAChE,IAAI,SAAS,IAAI,SAAS,QAAQ,EAAE,CAEnC;IAED,OAAO,CAAC,YAAY;IAMpB;;;OAGG;IACH,IAAI,uBAAuB,IAAI,aAAa,EAAE,GAAG,SAAS,CAEzD;IAED,IAAI,uBAAuB,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,SAAS,EAE9D;IA6HD,OAAO,CAAC,kBAAkB;YAaZ,cAAc;IAmB5B;;;;OAIG;IACG,iBAAiB,CACrB,UAAU,EAAE,oBAAoB,EAChC,MAAM,EAAE,yCAAyC,GAChD,OAAO,CAAC,2BAA2B,CAAC;IAwGjC,iBAAiB,CAAC,QAAQ,EAAE,EAChC,MAAM,EACN,UAAU,EACV,SAAa,GACd,EAAE;QACD,MAAM,EAAE,yCAAyC,CAAC;QAClD,UAAU,CAAC,EAAE,kCAAkC,CAC7C,cAAc,EACd,oBAAoB,EAAE,EACtB,QAAQ,CACT,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,UAAU,CAAC;CAsHxB"}