@devrev/ts-adaas 0.0.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/README.md +186 -116
  2. package/dist/common/constants.d.ts +10 -0
  3. package/dist/common/constants.js +27 -0
  4. package/dist/common/control-protocol.d.ts +7 -0
  5. package/dist/common/control-protocol.js +41 -0
  6. package/dist/common/helpers.d.ts +4 -0
  7. package/dist/common/helpers.js +38 -0
  8. package/dist/common/install-initial-domain-mapping.d.ts +3 -0
  9. package/dist/common/install-initial-domain-mapping.js +87 -0
  10. package/dist/{src → deprecated}/adapter/index.d.ts +10 -7
  11. package/dist/{src → deprecated}/adapter/index.js +13 -10
  12. package/dist/{src → deprecated}/common/helpers.d.ts +1 -2
  13. package/dist/deprecated/common/helpers.js +47 -0
  14. package/dist/deprecated/demo-extractor/external_domain_metadata.json +38 -0
  15. package/dist/deprecated/demo-extractor/index.d.ts +17 -0
  16. package/dist/deprecated/demo-extractor/index.js +161 -0
  17. package/dist/{src → deprecated}/uploader/index.d.ts +1 -1
  18. package/dist/{src → deprecated}/uploader/index.js +3 -1
  19. package/dist/{src/http → http}/client.d.ts +2 -1
  20. package/dist/index.d.ts +10 -0
  21. package/dist/index.js +30 -0
  22. package/dist/logger/logger.d.ts +14 -0
  23. package/dist/logger/logger.interfaces.d.ts +14 -0
  24. package/dist/logger/logger.interfaces.js +9 -0
  25. package/dist/logger/logger.js +91 -0
  26. package/dist/logger/logger.test.js +49 -0
  27. package/dist/repo/repo.d.ts +14 -0
  28. package/dist/repo/repo.interfaces.d.ts +43 -0
  29. package/dist/repo/repo.interfaces.js +2 -0
  30. package/dist/repo/repo.js +68 -0
  31. package/dist/repo/repo.test.js +74 -0
  32. package/dist/state/state.d.ts +24 -0
  33. package/dist/state/state.interfaces.d.ts +24 -0
  34. package/dist/state/state.interfaces.js +2 -0
  35. package/dist/state/state.js +115 -0
  36. package/dist/tests/test-helpers.d.ts +9 -0
  37. package/dist/tests/test-helpers.interfaces.d.ts +11 -0
  38. package/dist/tests/test-helpers.interfaces.js +2 -0
  39. package/dist/tests/test-helpers.js +88 -0
  40. package/dist/tests/test-worker.js +16 -0
  41. package/dist/types/common.d.ts +39 -0
  42. package/dist/{src/types → types}/common.js +4 -0
  43. package/dist/{src/types → types}/extraction.d.ts +57 -12
  44. package/dist/{src/types → types}/extraction.js +12 -29
  45. package/dist/types/index.d.ts +5 -0
  46. package/dist/types/index.js +11 -0
  47. package/dist/types/workers.d.ts +142 -0
  48. package/dist/types/workers.js +23 -0
  49. package/dist/uploader/uploader.d.ts +38 -0
  50. package/dist/uploader/uploader.interfaces.d.ts +63 -0
  51. package/dist/uploader/uploader.interfaces.js +2 -0
  52. package/dist/uploader/uploader.js +351 -0
  53. package/dist/{tests → uploader}/uploader.test.js +7 -5
  54. package/dist/workers/create-worker.d.ts +4 -0
  55. package/dist/workers/create-worker.js +30 -0
  56. package/dist/workers/create-worker.test.js +25 -0
  57. package/dist/workers/default-workers/attachments-deletion.d.ts +1 -0
  58. package/dist/workers/default-workers/attachments-deletion.js +13 -0
  59. package/dist/workers/default-workers/attachments-extraction.d.ts +1 -0
  60. package/dist/workers/default-workers/attachments-extraction.js +49 -0
  61. package/dist/workers/default-workers/data-deletion.d.ts +1 -0
  62. package/dist/workers/default-workers/data-deletion.js +15 -0
  63. package/dist/workers/default-workers/data-extraction.d.ts +1 -0
  64. package/dist/workers/default-workers/data-extraction.js +101 -0
  65. package/dist/workers/default-workers/external-sync-units-extraction.d.ts +1 -0
  66. package/dist/workers/default-workers/external-sync-units-extraction.js +27 -0
  67. package/dist/workers/default-workers/metadata-extraction.d.ts +1 -0
  68. package/dist/workers/default-workers/metadata-extraction.js +26 -0
  69. package/dist/workers/dummy-extractor/data-normalization.d.ts +4 -0
  70. package/dist/workers/dummy-extractor/data-normalization.js +41 -0
  71. package/dist/workers/dummy-extractor/external_domain_metadata.json +58 -0
  72. package/dist/workers/process-task.d.ts +2 -0
  73. package/dist/workers/process-task.js +44 -0
  74. package/dist/workers/spawn.d.ts +25 -0
  75. package/dist/workers/spawn.js +163 -0
  76. package/dist/workers/worker-adapter.d.ts +48 -0
  77. package/dist/workers/worker-adapter.js +138 -0
  78. package/dist/workers/worker.d.ts +1 -0
  79. package/dist/workers/worker.js +6 -0
  80. package/package.json +9 -5
  81. package/dist/src/common/constants.d.ts +0 -2
  82. package/dist/src/common/constants.js +0 -10
  83. package/dist/src/common/helpers.js +0 -59
  84. package/dist/src/demo-extractor/index.d.ts +0 -11
  85. package/dist/src/demo-extractor/index.js +0 -157
  86. package/dist/src/demo-extractor/initial_domain_mapping.json +0 -107
  87. package/dist/src/index.d.ts +0 -5
  88. package/dist/src/index.js +0 -21
  89. package/dist/src/logging/index.d.ts +0 -18
  90. package/dist/src/logging/index.js +0 -39
  91. package/dist/src/state/index.d.ts +0 -23
  92. package/dist/src/state/index.js +0 -111
  93. package/dist/src/types/common.d.ts +0 -33
  94. package/dist/src/types/index.d.ts +0 -2
  95. package/dist/src/types/index.js +0 -18
  96. package/dist/tests/adapter.helpers.test.js +0 -60
  97. package/dist/tests/adapter.test.js +0 -122
  98. package/dist/tests/demo-extractor.test.js +0 -60
  99. package/dist/tests/state.test.js +0 -100
  100. /package/dist/{src/http → http}/client.js +0 -0
  101. /package/dist/{src/http → http}/constants.d.ts +0 -0
  102. /package/dist/{src/http → http}/constants.js +0 -0
  103. /package/dist/{src/http → http}/index.d.ts +0 -0
  104. /package/dist/{src/http → http}/index.js +0 -0
  105. /package/dist/{src/http → http}/types.d.ts +0 -0
  106. /package/dist/{src/http → http}/types.js +0 -0
  107. /package/dist/{tests/adapter.helpers.test.d.ts → logger/logger.test.d.ts} +0 -0
  108. /package/dist/{tests/adapter.test.d.ts → repo/repo.test.d.ts} +0 -0
  109. /package/dist/tests/{demo-extractor.test.d.ts → test-worker.d.ts} +0 -0
  110. /package/dist/{tests → uploader}/uploader.test.d.ts +0 -0
  111. /package/dist/{tests/state.test.d.ts → workers/create-worker.test.d.ts} +0 -0
@@ -3,12 +3,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
3
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.Adapter = exports.createAdapter = void 0;
6
+ exports.Adapter = void 0;
7
+ exports.createAdapter = createAdapter;
7
8
  const axios_1 = __importDefault(require("axios"));
8
- const constants_1 = require("../common/constants");
9
+ const constants_1 = require("../../common/constants");
9
10
  const helpers_1 = require("../common/helpers");
10
- const logging_1 = require("../logging");
11
- const state_1 = require("../state");
11
+ // import { Logger } from '../../logger/logger';
12
+ const state_1 = require("../../state/state");
12
13
  /**
13
14
  * Adapter class is used to interact with Airdrop platform. The class provides
14
15
  * utilities to
@@ -18,6 +19,7 @@ const state_1 = require("../state");
18
19
  *
19
20
  * @class Adapter
20
21
  * @constructor
22
+ * @deprecated
21
23
  * @param {AirdropEvent} event - The event object received from the platform
22
24
  * @param {object=} initialState - The initial state of the adapter
23
25
  * @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development
@@ -32,19 +34,21 @@ const state_1 = require("../state");
32
34
  */
33
35
  async function createAdapter(event, initialState, isLocalDevelopment = false) {
34
36
  const newInitialState = structuredClone(initialState);
35
- const adapterState = await (0, state_1.createAdapterState)(event, newInitialState);
37
+ const adapterState = await (0, state_1.createAdapterState)({
38
+ event,
39
+ initialState: newInitialState,
40
+ });
36
41
  const a = new Adapter(event, adapterState, isLocalDevelopment);
37
42
  return a;
38
43
  }
39
- exports.createAdapter = createAdapter;
40
44
  class Adapter {
41
45
  constructor(event, adapterState, isLocalDevelopment = false) {
46
+ // if (!isLocalDevelopment) {
47
+ // Logger.init(event);
48
+ // }
42
49
  this.exit = false;
43
50
  this.lambdaTimeout = 10 * 60 * 1000; // 10 minutes in milliseconds
44
51
  this.heartBeatInterval = 30 * 1000; // 30 seconds in milliseconds
45
- if (!isLocalDevelopment) {
46
- logging_1.Logger.init(event);
47
- }
48
52
  this.adapterState = adapterState;
49
53
  this._artifacts = [];
50
54
  this.event = event;
@@ -122,7 +126,6 @@ class Adapter {
122
126
  */
123
127
  exitAdapter() {
124
128
  this.exit = true;
125
- clearInterval(this.heartBeatFn);
126
129
  }
127
130
  /**
128
131
  * Heartbeat function to check if the lambda is about to timeout.
@@ -1,6 +1,5 @@
1
- import { Artifact, EventType, ExtractorEventType } from '../types';
1
+ import { EventType, ExtractorEventType } from '../../types/extraction';
2
2
  export declare function createFormData(preparedArtifact: any, fetchedObjects: object[] | object): FormData;
3
- export declare function createArtifact(preparedArtifact: any, fetchedObjects: object[] | object, entity: string): Artifact;
4
3
  export declare function getTimeoutExtractorEventType(eventType: EventType): {
5
4
  eventType: ExtractorEventType;
6
5
  isError: boolean;
@@ -0,0 +1,47 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.createFormData = createFormData;
4
+ exports.getTimeoutExtractorEventType = getTimeoutExtractorEventType;
5
+ const js_jsonl_1 = require("js-jsonl");
6
+ const extraction_1 = require("../../types/extraction");
7
+ function createFormData(
8
+ //eslint-disable-next-line @typescript-eslint/no-explicit-any
9
+ preparedArtifact, fetchedObjects) {
10
+ const formData = new FormData();
11
+ for (const item of preparedArtifact.form_data) {
12
+ formData.append(item.key, item.value);
13
+ }
14
+ const output = js_jsonl_1.jsonl.stringify(fetchedObjects);
15
+ formData.append('file', output);
16
+ return formData;
17
+ }
18
+ function getTimeoutExtractorEventType(eventType) {
19
+ switch (eventType) {
20
+ case extraction_1.EventType.ExtractionMetadataStart:
21
+ return {
22
+ eventType: extraction_1.ExtractorEventType.ExtractionMetadataError,
23
+ isError: true,
24
+ };
25
+ case extraction_1.EventType.ExtractionDataStart:
26
+ case extraction_1.EventType.ExtractionDataContinue:
27
+ return {
28
+ eventType: extraction_1.ExtractorEventType.ExtractionDataProgress,
29
+ isError: false,
30
+ };
31
+ case extraction_1.EventType.ExtractionAttachmentsStart:
32
+ case extraction_1.EventType.ExtractionAttachmentsContinue:
33
+ return {
34
+ eventType: extraction_1.ExtractorEventType.ExtractionAttachmentsProgress,
35
+ isError: false,
36
+ };
37
+ case extraction_1.EventType.ExtractionExternalSyncUnitsStart:
38
+ return {
39
+ eventType: extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsError,
40
+ isError: true,
41
+ };
42
+ default:
43
+ console.log('Event type not recognized in getTimeoutExtractorEventType function: ' +
44
+ eventType);
45
+ return null;
46
+ }
47
+ }
@@ -0,0 +1,38 @@
1
+ {
2
+ "record_types": {
3
+ "users": {
4
+ "fields": {
5
+ "name": {
6
+ "is_required": true,
7
+ "type": "text",
8
+ "name": "Name",
9
+ "text": {
10
+ "min_length": 1
11
+ }
12
+ },
13
+ "email": {
14
+ "type": "text",
15
+ "name": "Email",
16
+ "is_required": true
17
+ }
18
+ }
19
+ },
20
+ "contacts": {
21
+ "fields": {
22
+ "name": {
23
+ "is_required": true,
24
+ "type": "text",
25
+ "name": "Name",
26
+ "text": {
27
+ "min_length": 1
28
+ }
29
+ },
30
+ "email": {
31
+ "type": "text",
32
+ "name": "Email",
33
+ "is_required": true
34
+ }
35
+ }
36
+ }
37
+ }
38
+ }
@@ -0,0 +1,17 @@
1
+ import { AirdropEvent } from '../../types/extraction';
2
+ import { Adapter } from '../adapter';
3
+ type ConnectorState = object;
4
+ /**
5
+ * Demo extractor is a reference implementation of an ADaaS connector to facilitate rapid immersion into ADaaS.
6
+ *
7
+ * @class DemoExtractor
8
+ * @deprecated
9
+ **/
10
+ export declare class DemoExtractor {
11
+ private event;
12
+ private adapter;
13
+ private uploader;
14
+ constructor(event: AirdropEvent, adapter: Adapter<ConnectorState>);
15
+ run(): Promise<void>;
16
+ }
17
+ export {};
@@ -0,0 +1,161 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.DemoExtractor = void 0;
7
+ const extraction_1 = require("../../types/extraction");
8
+ const uploader_1 = require("../uploader");
9
+ const external_domain_metadata_json_1 = __importDefault(require("./external_domain_metadata.json"));
10
+ /**
11
+ * Demo extractor is a reference implementation of an ADaaS connector to facilitate rapid immersion into ADaaS.
12
+ *
13
+ * @class DemoExtractor
14
+ * @deprecated
15
+ **/
16
+ class DemoExtractor {
17
+ constructor(event, adapter) {
18
+ this.event = event;
19
+ this.adapter = adapter;
20
+ this.uploader = new uploader_1.Uploader(this.event.execution_metadata.devrev_endpoint, this.event.context.secrets.service_account_token);
21
+ }
22
+ async run() {
23
+ switch (this.event.payload.event_type) {
24
+ case extraction_1.EventType.ExtractionExternalSyncUnitsStart: {
25
+ const externalSyncUnits = [
26
+ {
27
+ id: 'devrev',
28
+ name: 'devrev',
29
+ description: 'Demo external sync unit',
30
+ },
31
+ ];
32
+ await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionExternalSyncUnitsDone, {
33
+ external_sync_units: externalSyncUnits,
34
+ });
35
+ break;
36
+ }
37
+ case extraction_1.EventType.ExtractionMetadataStart: {
38
+ const { artifact, error } = await this.uploader.upload('metadata_1.jsonl', 'external_domain_metadata', external_domain_metadata_json_1.default);
39
+ if (error || !artifact) {
40
+ await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionMetadataError, {
41
+ error,
42
+ });
43
+ return;
44
+ }
45
+ await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionMetadataDone, {
46
+ artifacts: [artifact],
47
+ });
48
+ break;
49
+ }
50
+ case extraction_1.EventType.ExtractionDataStart: {
51
+ const contacts = [
52
+ {
53
+ id: 'contact-1',
54
+ created_date: '1999-12-25T01:00:03+01:00',
55
+ modified_date: '1999-12-25T01:00:03+01:00',
56
+ data: {
57
+ email: 'johnsmith@test.com',
58
+ name: 'John Smith',
59
+ },
60
+ },
61
+ {
62
+ id: 'contact-2',
63
+ created_date: '1999-12-27T15:31:34+01:00',
64
+ modified_date: '2002-04-09T01:55:31+02:00',
65
+ data: {
66
+ email: 'janesmith@test.com',
67
+ name: 'Jane Smith',
68
+ },
69
+ },
70
+ ];
71
+ const { artifact, error } = await this.uploader.upload('contacts_1.json', 'contacts', contacts);
72
+ if (error || !artifact) {
73
+ await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionDataError, {
74
+ error,
75
+ });
76
+ return;
77
+ }
78
+ await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionDataProgress, {
79
+ progress: 50,
80
+ artifacts: [artifact],
81
+ });
82
+ break;
83
+ }
84
+ case extraction_1.EventType.ExtractionDataContinue: {
85
+ const users = [
86
+ {
87
+ id: 'user-1',
88
+ created_date: '1999-12-25T01:00:03+01:00',
89
+ modified_date: '1999-12-25T01:00:03+01:00',
90
+ data: {
91
+ email: 'johndoe@test.com',
92
+ name: 'John Doe',
93
+ },
94
+ },
95
+ {
96
+ id: 'user-2',
97
+ created_date: '1999-12-27T15:31:34+01:00',
98
+ modified_date: '2002-04-09T01:55:31+02:00',
99
+ data: {
100
+ email: 'janedoe@test.com',
101
+ name: 'Jane Doe',
102
+ },
103
+ },
104
+ ];
105
+ const { artifact, error } = await this.uploader.upload('users_1.json', 'users', users);
106
+ if (error || !artifact) {
107
+ await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionDataError, {
108
+ error,
109
+ });
110
+ return;
111
+ }
112
+ await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionDataDone, {
113
+ progress: 100,
114
+ artifacts: [artifact],
115
+ });
116
+ break;
117
+ }
118
+ case extraction_1.EventType.ExtractionDataDelete: {
119
+ await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionDataDeleteDone);
120
+ break;
121
+ }
122
+ case extraction_1.EventType.ExtractionAttachmentsStart: {
123
+ const attachment1 = ['This is attachment1.txt content'];
124
+ const { artifact, error } = await this.uploader.upload('attachment1.txt', 'attachment', attachment1);
125
+ if (error || !artifact) {
126
+ await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionAttachmentsError, {
127
+ error,
128
+ });
129
+ return;
130
+ }
131
+ await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionAttachmentsProgress, {
132
+ artifacts: [artifact],
133
+ });
134
+ break;
135
+ }
136
+ case extraction_1.EventType.ExtractionAttachmentsContinue: {
137
+ const attachment2 = ['This is attachment2.txt content'];
138
+ const { artifact, error } = await this.uploader.upload('attachment2.txt', 'attachment', attachment2);
139
+ if (error || !artifact) {
140
+ await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionAttachmentsError, {
141
+ error,
142
+ });
143
+ return;
144
+ }
145
+ await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionAttachmentsDone, {
146
+ artifacts: [artifact],
147
+ });
148
+ break;
149
+ }
150
+ case extraction_1.EventType.ExtractionAttachmentsDelete: {
151
+ await this.adapter.emit(extraction_1.ExtractorEventType.ExtractionAttachmentsDeleteDone);
152
+ break;
153
+ }
154
+ default: {
155
+ console.error('Event in DemoExtractor run not recognized: ' +
156
+ JSON.stringify(this.event.payload.event_type));
157
+ }
158
+ }
159
+ }
160
+ }
161
+ exports.DemoExtractor = DemoExtractor;
@@ -1,4 +1,4 @@
1
- import { UploadResponse } from '../types/common';
1
+ import { UploadResponse } from '../../uploader/uploader.interfaces';
2
2
  /**
3
3
  * Uploader class is used to upload files to the DevRev platform.
4
4
  * The class provides utilities to
@@ -106,7 +106,9 @@ class Uploader {
106
106
  }
107
107
  async uploadToArtifact(
108
108
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
109
- preparedArtifact, fetchedObjects) {
109
+ preparedArtifact, fetchedObjects
110
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
111
+ ) {
110
112
  const formData = (0, helpers_1.createFormData)(preparedArtifact, fetchedObjects);
111
113
  try {
112
114
  const response = await axios_1.default.post(preparedArtifact.url, formData, {
@@ -1,3 +1,4 @@
1
+ import { RawAxiosRequestHeaders } from 'axios';
1
2
  import { HTTPResponse } from './types';
2
3
  export declare const defaultResponse: HTTPResponse;
3
4
  export declare class HTTPClient {
@@ -12,5 +13,5 @@ export declare class HTTPClient {
12
13
  * In case of rate limit exceeded, the function returns success as true and the delay time in seconds
13
14
  * In case of any other error, the function returns success as false and the error message
14
15
  */
15
- getCall(endpoint: string, headers: Record<string, string>, params?: any): Promise<HTTPResponse>;
16
+ getCall(endpoint: string, headers: RawAxiosRequestHeaders, params?: any): Promise<HTTPResponse>;
16
17
  }
@@ -0,0 +1,10 @@
1
+ export * from './deprecated/adapter';
2
+ export * from './deprecated/demo-extractor';
3
+ export * from './deprecated/uploader';
4
+ export * from './types';
5
+ export * from './http';
6
+ export * from './common/install-initial-domain-mapping';
7
+ export { WorkerAdapter } from './workers/worker-adapter';
8
+ export { processTask } from './workers/process-task';
9
+ export { spawn } from './workers/spawn';
10
+ export * from './types/workers';
package/dist/index.js ADDED
@@ -0,0 +1,30 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
+ };
16
+ Object.defineProperty(exports, "__esModule", { value: true });
17
+ exports.spawn = exports.processTask = exports.WorkerAdapter = void 0;
18
+ __exportStar(require("./deprecated/adapter"), exports);
19
+ __exportStar(require("./deprecated/demo-extractor"), exports);
20
+ __exportStar(require("./deprecated/uploader"), exports);
21
+ __exportStar(require("./types"), exports);
22
+ __exportStar(require("./http"), exports);
23
+ __exportStar(require("./common/install-initial-domain-mapping"), exports);
24
+ var worker_adapter_1 = require("./workers/worker-adapter");
25
+ Object.defineProperty(exports, "WorkerAdapter", { enumerable: true, get: function () { return worker_adapter_1.WorkerAdapter; } });
26
+ var process_task_1 = require("./workers/process-task");
27
+ Object.defineProperty(exports, "processTask", { enumerable: true, get: function () { return process_task_1.processTask; } });
28
+ var spawn_1 = require("./workers/spawn");
29
+ Object.defineProperty(exports, "spawn", { enumerable: true, get: function () { return spawn_1.spawn; } });
30
+ __exportStar(require("./types/workers"), exports);
@@ -0,0 +1,14 @@
1
+ import { Console } from 'node:console';
2
+ import { LogLevel, PrintableState } from './logger.interfaces';
3
+ import { AirdropEvent } from '../types';
4
+ import { AxiosError } from 'axios';
5
+ export declare class Logger extends Console {
6
+ constructor(event: AirdropEvent);
7
+ logFn(args: unknown[], level: LogLevel): void;
8
+ log(...args: unknown[]): void;
9
+ info(...args: unknown[]): void;
10
+ warn(...args: unknown[]): void;
11
+ error(...args: unknown[]): void;
12
+ }
13
+ export declare function getPrintableState(state: Record<string, any>): PrintableState;
14
+ export declare function formatAxiosError(error: AxiosError): object;
@@ -0,0 +1,14 @@
1
+ export declare enum LogLevel {
2
+ INFO = "info",
3
+ WARN = "warn",
4
+ ERROR = "error"
5
+ }
6
+ export interface PrintableArray {
7
+ type: 'array';
8
+ length: number;
9
+ firstItem?: any;
10
+ lastItem?: any;
11
+ }
12
+ export interface PrintableState {
13
+ [key: string]: any | PrintableArray | PrintableState;
14
+ }
@@ -0,0 +1,9 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.LogLevel = void 0;
4
+ var LogLevel;
5
+ (function (LogLevel) {
6
+ LogLevel["INFO"] = "info";
7
+ LogLevel["WARN"] = "warn";
8
+ LogLevel["ERROR"] = "error";
9
+ })(LogLevel || (exports.LogLevel = LogLevel = {}));
@@ -0,0 +1,91 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.Logger = void 0;
7
+ exports.getPrintableState = getPrintableState;
8
+ exports.formatAxiosError = formatAxiosError;
9
+ /* eslint-disable @typescript-eslint/no-explicit-any */
10
+ const lambda_log_1 = __importDefault(require("lambda-log"));
11
+ const node_console_1 = require("node:console");
12
+ const logger_interfaces_1 = require("./logger.interfaces");
13
+ const node_worker_threads_1 = require("node:worker_threads");
14
+ const workers_1 = require("../types/workers");
15
+ class Logger extends node_console_1.Console {
16
+ constructor(event) {
17
+ super(process.stdout, process.stderr);
18
+ lambda_log_1.default.options.levelKey = null;
19
+ lambda_log_1.default.options.tagsKey = null;
20
+ lambda_log_1.default.options.messageKey = 'message';
21
+ lambda_log_1.default.options.meta = Object.assign({}, event.payload.event_context);
22
+ }
23
+ logFn(args, level) {
24
+ if (node_worker_threads_1.isMainThread) {
25
+ lambda_log_1.default.log(level, JSON.stringify(args));
26
+ }
27
+ else {
28
+ node_worker_threads_1.parentPort === null || node_worker_threads_1.parentPort === void 0 ? void 0 : node_worker_threads_1.parentPort.postMessage({
29
+ subject: workers_1.WorkerMessageSubject.WorkerMessageLog,
30
+ payload: {
31
+ args,
32
+ level,
33
+ },
34
+ });
35
+ }
36
+ }
37
+ log(...args) {
38
+ this.logFn(args, logger_interfaces_1.LogLevel.INFO);
39
+ }
40
+ info(...args) {
41
+ this.logFn(args, logger_interfaces_1.LogLevel.INFO);
42
+ }
43
+ warn(...args) {
44
+ this.logFn(args, logger_interfaces_1.LogLevel.WARN);
45
+ }
46
+ error(...args) {
47
+ this.logFn(args, logger_interfaces_1.LogLevel.ERROR);
48
+ }
49
+ }
50
+ exports.Logger = Logger;
51
+ // Helper function to process each value in the state
52
+ function getPrintableState(state) {
53
+ function processValue(value) {
54
+ if (Array.isArray(value)) {
55
+ // If the value is an array, summarize it
56
+ return {
57
+ type: 'array',
58
+ length: value.length,
59
+ firstItem: value.length > 0 ? value[0] : undefined,
60
+ lastItem: value.length > 1 ? value[value.length - 1] : undefined,
61
+ };
62
+ }
63
+ else if (typeof value === 'object' && value !== null) {
64
+ // If the value is an object, recursively process its properties
65
+ const processedObject = {};
66
+ for (const key in value) {
67
+ if (value.hasOwnProperty(key)) {
68
+ processedObject[key] = processValue(value[key]);
69
+ }
70
+ }
71
+ return processedObject;
72
+ }
73
+ // For primitive types, return the value as is
74
+ return value;
75
+ }
76
+ // Process the state object directly since it's guaranteed to be an object
77
+ return processValue(state);
78
+ }
79
+ function formatAxiosError(error) {
80
+ var _a, _b, _c;
81
+ if (error.response) {
82
+ return {
83
+ status: error.response.status,
84
+ data: error.response.data,
85
+ method: (_a = error.config) === null || _a === void 0 ? void 0 : _a.method,
86
+ baseURL: (_b = error.config) === null || _b === void 0 ? void 0 : _b.baseURL,
87
+ url: (_c = error.config) === null || _c === void 0 ? void 0 : _c.url,
88
+ };
89
+ }
90
+ return error;
91
+ }
@@ -0,0 +1,49 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ const logger_1 = require("./logger");
4
+ it('getPrintableState should return printable state', () => {
5
+ const state = {
6
+ test_key: 'test_value',
7
+ big_array: Array.from({ length: 1000 }, (_, index) => index),
8
+ nested_object: {
9
+ nested_key: 'nested_value',
10
+ nested_array: Array.from({ length: 1000 }, (_, index) => index),
11
+ },
12
+ };
13
+ const printableState = (0, logger_1.getPrintableState)(state);
14
+ expect(printableState).toEqual({
15
+ test_key: 'test_value',
16
+ big_array: {
17
+ type: 'array',
18
+ length: 1000,
19
+ firstItem: 0,
20
+ lastItem: 999,
21
+ },
22
+ nested_object: {
23
+ nested_key: 'nested_value',
24
+ nested_array: {
25
+ type: 'array',
26
+ length: 1000,
27
+ firstItem: 0,
28
+ lastItem: 999,
29
+ },
30
+ },
31
+ });
32
+ });
33
+ it('formatAxiosError should return formatted error', () => {
34
+ const error = {
35
+ response: {
36
+ status: 500,
37
+ data: 'Internal server error',
38
+ },
39
+ config: {
40
+ method: 'GET',
41
+ },
42
+ };
43
+ const formattedError = (0, logger_1.formatAxiosError)(error);
44
+ expect(formattedError).toEqual({
45
+ status: 500,
46
+ data: 'Internal server error',
47
+ method: 'GET',
48
+ });
49
+ });
@@ -0,0 +1,14 @@
1
+ import { ErrorRecord } from '../types/common';
2
+ import { Item } from '../repo/repo.interfaces';
3
+ import { RepoFactoryInterface, NormalizedItem, NormalizedAttachment } from './repo.interfaces';
4
+ export declare class Repo {
5
+ readonly itemType: string;
6
+ private items;
7
+ private normalize?;
8
+ private uploader;
9
+ private onUpload;
10
+ constructor({ event, itemType, normalize, onUpload, options, }: RepoFactoryInterface);
11
+ getItems(): (NormalizedItem | NormalizedAttachment | Item)[];
12
+ upload(batch?: (NormalizedItem | NormalizedAttachment | Item)[]): Promise<void | ErrorRecord>;
13
+ push(items: Item[]): Promise<void | ErrorRecord>;
14
+ }
@@ -0,0 +1,43 @@
1
+ import { Artifact } from '../uploader/uploader.interfaces';
2
+ import { AirdropEvent } from '../types/extraction';
3
+ import { WorkerAdapterOptions } from '../types/workers';
4
+ /**
5
+ * RepoInterface is an interface that defines the structure of a repo which is used to store and upload extracted data.
6
+ */
7
+ export interface RepoInterface {
8
+ itemType: string;
9
+ normalize?: (record: object) => NormalizedItem | NormalizedAttachment;
10
+ }
11
+ /**
12
+ * RepoFactoryInterface is an interface that defines the structure of a repo factory which is used to create a repo.
13
+ */
14
+ export interface RepoFactoryInterface {
15
+ event: AirdropEvent;
16
+ itemType: string;
17
+ normalize?: (record: object) => NormalizedItem | NormalizedAttachment;
18
+ onUpload: (artifact: Artifact) => void;
19
+ options?: WorkerAdapterOptions;
20
+ }
21
+ /**
22
+ * NormalizedItem is an interface of item after normalization.
23
+ */
24
+ export interface NormalizedItem {
25
+ id: string;
26
+ created_date: string;
27
+ modified_date: string;
28
+ data: object;
29
+ }
30
+ /**
31
+ * NormalizedAttachment is an interface of attachment after normalization.
32
+ */
33
+ export interface NormalizedAttachment {
34
+ url: string;
35
+ id: string;
36
+ file_name: string;
37
+ author_id: string;
38
+ parent_id: string;
39
+ }
40
+ /**
41
+ * Item is an interface that defines the structure of an item.
42
+ */
43
+ export type Item = Record<string, any>;
@@ -0,0 +1,2 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });