@volley/sony-music-ingestion 1.0.72
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/constants.d.ts +5 -0
- package/dist/config/constants.d.ts.map +1 -0
- package/dist/config/constants.js +7 -0
- package/dist/config/setup-localstack.js +310 -0
- package/dist/core/ErnManager.d.ts +59 -0
- package/dist/core/ErnManager.d.ts.map +1 -0
- package/dist/core/ErnManager.js +235 -0
- package/dist/core/ErnManager.spec.d.ts +2 -0
- package/dist/core/ErnManager.spec.d.ts.map +1 -0
- package/dist/core/ErnManager.spec.js +297 -0
- package/dist/core/ErnParser.d.ts +67 -0
- package/dist/core/ErnParser.d.ts.map +1 -0
- package/dist/core/ErnParser.js +133 -0
- package/dist/core/ErnParser.spec.d.ts +2 -0
- package/dist/core/ErnParser.spec.d.ts.map +1 -0
- package/dist/core/ErnParser.spec.js +68 -0
- package/dist/core/TokenManager.d.ts +42 -0
- package/dist/core/TokenManager.d.ts.map +1 -0
- package/dist/core/TokenManager.js +92 -0
- package/dist/core/TokenManager.spec.d.ts +2 -0
- package/dist/core/TokenManager.spec.d.ts.map +1 -0
- package/dist/core/TokenManager.spec.js +232 -0
- package/dist/http/BackoffHttpClient.d.ts +18 -0
- package/dist/http/BackoffHttpClient.d.ts.map +1 -0
- package/dist/http/BackoffHttpClient.js +61 -0
- package/dist/http/BackoffHttpClient.spec.d.ts +2 -0
- package/dist/http/BackoffHttpClient.spec.d.ts.map +1 -0
- package/dist/http/BackoffHttpClient.spec.js +79 -0
- package/dist/http/ErrorHandler.d.ts +7 -0
- package/dist/http/ErrorHandler.d.ts.map +1 -0
- package/dist/http/ErrorHandler.js +16 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +102 -0
- package/dist/lib/index.d.ts +9 -0
- package/dist/lib/index.d.ts.map +1 -0
- package/dist/lib/index.js +16 -0
- package/dist/storage/dynamoDb/DealStorage.d.ts +99 -0
- package/dist/storage/dynamoDb/DealStorage.d.ts.map +1 -0
- package/dist/storage/dynamoDb/DealStorage.js +243 -0
- package/dist/storage/dynamoDb/DealStorage.spec.d.ts +2 -0
- package/dist/storage/dynamoDb/DealStorage.spec.d.ts.map +1 -0
- package/dist/storage/dynamoDb/DealStorage.spec.js +279 -0
- package/dist/storage/dynamoDb/ReleaseStorage.d.ts +37 -0
- package/dist/storage/dynamoDb/ReleaseStorage.d.ts.map +1 -0
- package/dist/storage/dynamoDb/ReleaseStorage.js +210 -0
- package/dist/storage/dynamoDb/ReleaseStorage.spec.d.ts +2 -0
- package/dist/storage/dynamoDb/ReleaseStorage.spec.d.ts.map +1 -0
- package/dist/storage/dynamoDb/ReleaseStorage.spec.js +251 -0
- package/dist/storage/dynamoDb/UnprocessedMessageStorage.d.ts +38 -0
- package/dist/storage/dynamoDb/UnprocessedMessageStorage.d.ts.map +1 -0
- package/dist/storage/dynamoDb/UnprocessedMessageStorage.js +117 -0
- package/dist/storage/dynamoDb/UnprocessedMessageStorage.spec.d.ts +2 -0
- package/dist/storage/dynamoDb/UnprocessedMessageStorage.spec.d.ts.map +1 -0
- package/dist/storage/dynamoDb/UnprocessedMessageStorage.spec.js +185 -0
- package/dist/storage/s3/AtomFeed.js +55 -0
- package/dist/storage/s3/AtomFeed.spec.d.ts +2 -0
- package/dist/storage/s3/AtomFeed.spec.d.ts.map +1 -0
- package/dist/storage/s3/AtomFeed.spec.js +65 -0
- package/dist/storage/s3/AtomFeedStorage.d.ts +19 -0
- package/dist/storage/s3/AtomFeedStorage.d.ts.map +1 -0
- package/dist/storage/s3/AtomFeedStorage.js +59 -0
- package/dist/storage/s3/Message.js +85 -0
- package/dist/storage/s3/Message.spec.d.ts +2 -0
- package/dist/storage/s3/Message.spec.d.ts.map +1 -0
- package/dist/storage/s3/Message.spec.js +57 -0
- package/dist/storage/s3/MessageStorage.d.ts +26 -0
- package/dist/storage/s3/MessageStorage.d.ts.map +1 -0
- package/dist/storage/s3/MessageStorage.js +89 -0
- package/dist/storage/s3/ResourceStorage.d.ts +27 -0
- package/dist/storage/s3/ResourceStorage.d.ts.map +1 -0
- package/dist/storage/s3/ResourceStorage.js +119 -0
- package/dist/storage/s3/ResourceStorage.spec.d.ts +2 -0
- package/dist/storage/s3/ResourceStorage.spec.d.ts.map +1 -0
- package/dist/storage/s3/ResourceStorage.spec.js +170 -0
- package/dist/test/utils/fakes/dynamodb/FakeDynamoDBClient.d.ts +21 -0
- package/dist/test/utils/fakes/dynamodb/FakeDynamoDBClient.d.ts.map +1 -0
- package/dist/test/utils/fakes/dynamodb/FakeDynamoDBClient.js +89 -0
- package/dist/test/utils/fakes/s3/FakeS3Client.d.ts +16 -0
- package/dist/test/utils/fakes/s3/FakeS3Client.d.ts.map +1 -0
- package/dist/test/utils/fakes/s3/FakeS3Client.js +59 -0
- package/dist/tools/extractTracks.d.ts +2 -0
- package/dist/tools/extractTracks.d.ts.map +1 -0
- package/dist/tools/extractTracks.js +24 -0
- package/dist/utils/arrays/conversion.d.ts +2 -0
- package/dist/utils/arrays/conversion.d.ts.map +1 -0
- package/dist/utils/arrays/conversion.js +10 -0
- package/dist/utils/compression/BrotliCompressor.d.ts +18 -0
- package/dist/utils/compression/BrotliCompressor.d.ts.map +1 -0
- package/dist/utils/compression/BrotliCompressor.js +65 -0
- package/dist/utils/compression/BrotliCompressor.spec.d.ts +2 -0
- package/dist/utils/compression/BrotliCompressor.spec.d.ts.map +1 -0
- package/dist/utils/compression/BrotliCompressor.spec.js +68 -0
- package/dist/utils/compression/test.js +19 -0
- package/dist/utils/compression/types.d.ts +44 -0
- package/dist/utils/compression/types.d.ts.map +1 -0
- package/dist/utils/compression/types.js +2 -0
- package/dist/utils/message/message.d.ts +3 -0
- package/dist/utils/message/message.d.ts.map +1 -0
- package/dist/utils/message/message.js +5 -0
- package/package.json +46 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/config/constants.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,eAAO,MAAM,eAAe,uBAAuB,CAAC"}
|
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
+
exports.setupLocalStack = void 0;
|
|
27
|
+
const aws_sdk_1 = require("aws-sdk");
|
|
28
|
+
const ReleaseStorage_1 = require("../storage/dynamoDb/ReleaseStorage");
|
|
29
|
+
const AtomFeedStorage_1 = require("../storage/s3/AtomFeedStorage");
|
|
30
|
+
const MessageStorage_1 = require("../storage/s3/MessageStorage");
|
|
31
|
+
const ResourceStorage_1 = require("../storage/s3/ResourceStorage");
|
|
32
|
+
const DealStorage_1 = require("../storage/dynamoDb/DealStorage");
|
|
33
|
+
const UnprocessedMessageStorage_1 = require("../storage/dynamoDb/UnprocessedMessageStorage");
|
|
34
|
+
const child_process_1 = require("child_process");
|
|
35
|
+
const http = __importStar(require("http"));
|
|
36
|
+
const ENDPOINT = "http://localhost:4566";
|
|
37
|
+
const REGION = "us-east-1";
|
|
38
|
+
async function checkHealth() {
|
|
39
|
+
return new Promise((resolve) => {
|
|
40
|
+
const req = http.get(`${ENDPOINT}/_localstack/health`, (res) => {
|
|
41
|
+
resolve(res.statusCode === 200);
|
|
42
|
+
});
|
|
43
|
+
req.on("error", () => resolve(false));
|
|
44
|
+
req.setTimeout(2000, () => {
|
|
45
|
+
req.destroy();
|
|
46
|
+
resolve(false);
|
|
47
|
+
});
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
async function startLocalStack() {
|
|
51
|
+
console.log("Starting LocalStack...");
|
|
52
|
+
const child = (0, child_process_1.spawn)("localstack", ["start", "-d"], {
|
|
53
|
+
detached: true,
|
|
54
|
+
stdio: "ignore",
|
|
55
|
+
});
|
|
56
|
+
child.unref();
|
|
57
|
+
for (let i = 0; i < 30; i++) {
|
|
58
|
+
await new Promise((resolve) => setTimeout(resolve, 1000));
|
|
59
|
+
if (await checkHealth()) {
|
|
60
|
+
console.log("Started LocalStack\n");
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
throw new Error("Failed to start LocalStack");
|
|
65
|
+
}
|
|
66
|
+
async function ensureRunning() {
|
|
67
|
+
try {
|
|
68
|
+
(0, child_process_1.execSync)("which localstack", { stdio: "ignore" });
|
|
69
|
+
}
|
|
70
|
+
catch {
|
|
71
|
+
throw new Error("LocalStack not installed. Install: brew install localstack/tap/localstack-cli");
|
|
72
|
+
}
|
|
73
|
+
if (!(await checkHealth())) {
|
|
74
|
+
await startLocalStack();
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
async function ensureBucket(name) {
|
|
78
|
+
try {
|
|
79
|
+
(0, child_process_1.execSync)(`aws s3 mb s3://${name} --endpoint-url ${ENDPOINT} --region ${REGION}`, { stdio: "ignore" });
|
|
80
|
+
console.log(name + " ready");
|
|
81
|
+
}
|
|
82
|
+
catch (error) {
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
async function ensureReleasesTable(dynamodb) {
|
|
86
|
+
try {
|
|
87
|
+
await dynamodb
|
|
88
|
+
.describeTable({ TableName: ReleaseStorage_1.ReleaseStorage.TABLE_NAME })
|
|
89
|
+
.promise();
|
|
90
|
+
console.log(ReleaseStorage_1.ReleaseStorage.TABLE_NAME + "ready");
|
|
91
|
+
}
|
|
92
|
+
catch (error) {
|
|
93
|
+
if (error.code === "ResourceNotFoundException") {
|
|
94
|
+
await dynamodb
|
|
95
|
+
.createTable({
|
|
96
|
+
TableName: ReleaseStorage_1.ReleaseStorage.TABLE_NAME,
|
|
97
|
+
AttributeDefinitions: [
|
|
98
|
+
{ AttributeName: "GRid", AttributeType: "S" },
|
|
99
|
+
{ AttributeName: "MessageId", AttributeType: "S" },
|
|
100
|
+
{ AttributeName: "ISRC", AttributeType: "S" },
|
|
101
|
+
{ AttributeName: "Title", AttributeType: "S" },
|
|
102
|
+
{ AttributeName: "CatalogNumber", AttributeType: "S" },
|
|
103
|
+
{ AttributeName: "ICPN", AttributeType: "S" },
|
|
104
|
+
{ AttributeName: "Artist", AttributeType: "S" },
|
|
105
|
+
],
|
|
106
|
+
KeySchema: [{ AttributeName: "GRid", KeyType: "HASH" }],
|
|
107
|
+
GlobalSecondaryIndexes: [
|
|
108
|
+
{
|
|
109
|
+
IndexName: "MessageId-index",
|
|
110
|
+
KeySchema: [{ AttributeName: "MessageId", KeyType: "HASH" }],
|
|
111
|
+
Projection: { ProjectionType: "ALL" },
|
|
112
|
+
ProvisionedThroughput: {
|
|
113
|
+
ReadCapacityUnits: 5,
|
|
114
|
+
WriteCapacityUnits: 5,
|
|
115
|
+
},
|
|
116
|
+
},
|
|
117
|
+
{
|
|
118
|
+
IndexName: "ISRC-index",
|
|
119
|
+
KeySchema: [{ AttributeName: "ISRC", KeyType: "HASH" }],
|
|
120
|
+
Projection: { ProjectionType: "ALL" },
|
|
121
|
+
ProvisionedThroughput: {
|
|
122
|
+
ReadCapacityUnits: 5,
|
|
123
|
+
WriteCapacityUnits: 5,
|
|
124
|
+
},
|
|
125
|
+
},
|
|
126
|
+
{
|
|
127
|
+
IndexName: "Title-index",
|
|
128
|
+
KeySchema: [{ AttributeName: "Title", KeyType: "HASH" }],
|
|
129
|
+
Projection: { ProjectionType: "ALL" },
|
|
130
|
+
ProvisionedThroughput: {
|
|
131
|
+
ReadCapacityUnits: 5,
|
|
132
|
+
WriteCapacityUnits: 5,
|
|
133
|
+
},
|
|
134
|
+
},
|
|
135
|
+
{
|
|
136
|
+
IndexName: "CatalogNumber-index",
|
|
137
|
+
KeySchema: [{ AttributeName: "CatalogNumber", KeyType: "HASH" }],
|
|
138
|
+
Projection: { ProjectionType: "ALL" },
|
|
139
|
+
ProvisionedThroughput: {
|
|
140
|
+
ReadCapacityUnits: 5,
|
|
141
|
+
WriteCapacityUnits: 5,
|
|
142
|
+
},
|
|
143
|
+
},
|
|
144
|
+
{
|
|
145
|
+
IndexName: "ICPN-index",
|
|
146
|
+
KeySchema: [{ AttributeName: "ICPN", KeyType: "HASH" }],
|
|
147
|
+
Projection: { ProjectionType: "ALL" },
|
|
148
|
+
ProvisionedThroughput: {
|
|
149
|
+
ReadCapacityUnits: 5,
|
|
150
|
+
WriteCapacityUnits: 5,
|
|
151
|
+
},
|
|
152
|
+
},
|
|
153
|
+
{
|
|
154
|
+
IndexName: "Artist-Title-index",
|
|
155
|
+
KeySchema: [
|
|
156
|
+
{ AttributeName: "Artist", KeyType: "HASH" },
|
|
157
|
+
{ AttributeName: "Title", KeyType: "RANGE" },
|
|
158
|
+
],
|
|
159
|
+
Projection: { ProjectionType: "ALL" },
|
|
160
|
+
ProvisionedThroughput: {
|
|
161
|
+
ReadCapacityUnits: 5,
|
|
162
|
+
WriteCapacityUnits: 5,
|
|
163
|
+
},
|
|
164
|
+
},
|
|
165
|
+
],
|
|
166
|
+
ProvisionedThroughput: {
|
|
167
|
+
ReadCapacityUnits: 5,
|
|
168
|
+
WriteCapacityUnits: 5,
|
|
169
|
+
},
|
|
170
|
+
})
|
|
171
|
+
.promise();
|
|
172
|
+
await dynamodb
|
|
173
|
+
.waitFor("tableExists", { TableName: ReleaseStorage_1.ReleaseStorage.TABLE_NAME })
|
|
174
|
+
.promise();
|
|
175
|
+
}
|
|
176
|
+
else {
|
|
177
|
+
throw error;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
async function ensureDealsTable(dynamodb) {
|
|
182
|
+
try {
|
|
183
|
+
await dynamodb
|
|
184
|
+
.describeTable({ TableName: DealStorage_1.DealStorage.TABLE_NAME })
|
|
185
|
+
.promise();
|
|
186
|
+
console.log(DealStorage_1.DealStorage.TABLE_NAME + "ready");
|
|
187
|
+
}
|
|
188
|
+
catch (error) {
|
|
189
|
+
if (error.code === "ResourceNotFoundException") {
|
|
190
|
+
await dynamodb
|
|
191
|
+
.createTable({
|
|
192
|
+
TableName: DealStorage_1.DealStorage.TABLE_NAME,
|
|
193
|
+
AttributeDefinitions: [
|
|
194
|
+
{ AttributeName: "isrc", AttributeType: "S" },
|
|
195
|
+
{ AttributeName: "dealHash", AttributeType: "S" },
|
|
196
|
+
],
|
|
197
|
+
KeySchema: [
|
|
198
|
+
{ AttributeName: "isrc", KeyType: "HASH" },
|
|
199
|
+
{ AttributeName: "dealHash", KeyType: "RANGE" },
|
|
200
|
+
],
|
|
201
|
+
ProvisionedThroughput: {
|
|
202
|
+
ReadCapacityUnits: 5,
|
|
203
|
+
WriteCapacityUnits: 5,
|
|
204
|
+
},
|
|
205
|
+
})
|
|
206
|
+
.promise();
|
|
207
|
+
await dynamodb
|
|
208
|
+
.waitFor("tableExists", { TableName: DealStorage_1.DealStorage.TABLE_NAME })
|
|
209
|
+
.promise();
|
|
210
|
+
}
|
|
211
|
+
else {
|
|
212
|
+
throw error;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
async function ensureUnprocessedMessagesTable(dynamodb) {
|
|
217
|
+
try {
|
|
218
|
+
await dynamodb
|
|
219
|
+
.describeTable({ TableName: UnprocessedMessageStorage_1.UnprocessedMessageStorage.TABLE_NAME })
|
|
220
|
+
.promise();
|
|
221
|
+
console.log(UnprocessedMessageStorage_1.UnprocessedMessageStorage.TABLE_NAME + " ready");
|
|
222
|
+
}
|
|
223
|
+
catch (error) {
|
|
224
|
+
if (error.code === "ResourceNotFoundException") {
|
|
225
|
+
await dynamodb
|
|
226
|
+
.createTable({
|
|
227
|
+
TableName: UnprocessedMessageStorage_1.UnprocessedMessageStorage.TABLE_NAME,
|
|
228
|
+
AttributeDefinitions: [
|
|
229
|
+
{ AttributeName: "type", AttributeType: "S" },
|
|
230
|
+
{ AttributeName: "timestamp", AttributeType: "S" },
|
|
231
|
+
{ AttributeName: "uuid", AttributeType: "S" },
|
|
232
|
+
],
|
|
233
|
+
KeySchema: [
|
|
234
|
+
{ AttributeName: "type", KeyType: "HASH" },
|
|
235
|
+
{ AttributeName: "timestamp", KeyType: "RANGE" },
|
|
236
|
+
],
|
|
237
|
+
GlobalSecondaryIndexes: [
|
|
238
|
+
{
|
|
239
|
+
IndexName: "uuid-index",
|
|
240
|
+
KeySchema: [{ AttributeName: "uuid", KeyType: "HASH" }],
|
|
241
|
+
Projection: { ProjectionType: "KEYS_ONLY" },
|
|
242
|
+
ProvisionedThroughput: {
|
|
243
|
+
ReadCapacityUnits: 5,
|
|
244
|
+
WriteCapacityUnits: 5,
|
|
245
|
+
},
|
|
246
|
+
},
|
|
247
|
+
{
|
|
248
|
+
IndexName: "uuid-timestamp-index",
|
|
249
|
+
KeySchema: [
|
|
250
|
+
{ AttributeName: "uuid", KeyType: "HASH" },
|
|
251
|
+
{ AttributeName: "timestamp", KeyType: "RANGE" },
|
|
252
|
+
],
|
|
253
|
+
Projection: { ProjectionType: "ALL" },
|
|
254
|
+
ProvisionedThroughput: {
|
|
255
|
+
ReadCapacityUnits: 5,
|
|
256
|
+
WriteCapacityUnits: 5,
|
|
257
|
+
},
|
|
258
|
+
},
|
|
259
|
+
],
|
|
260
|
+
ProvisionedThroughput: {
|
|
261
|
+
ReadCapacityUnits: 5,
|
|
262
|
+
WriteCapacityUnits: 5,
|
|
263
|
+
},
|
|
264
|
+
})
|
|
265
|
+
.promise();
|
|
266
|
+
await dynamodb
|
|
267
|
+
.waitFor("tableExists", { TableName: UnprocessedMessageStorage_1.UnprocessedMessageStorage.TABLE_NAME })
|
|
268
|
+
.promise();
|
|
269
|
+
}
|
|
270
|
+
else {
|
|
271
|
+
throw error;
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
async function setupLocalStack() {
|
|
276
|
+
const startTime = Date.now();
|
|
277
|
+
try {
|
|
278
|
+
await ensureRunning();
|
|
279
|
+
console.log(`LocalStack running (${Date.now() - startTime}ms)`);
|
|
280
|
+
const dynamodb = new aws_sdk_1.DynamoDB({
|
|
281
|
+
accessKeyId: "test",
|
|
282
|
+
secretAccessKey: "test",
|
|
283
|
+
endpoint: ENDPOINT,
|
|
284
|
+
region: REGION,
|
|
285
|
+
});
|
|
286
|
+
const buckets = [
|
|
287
|
+
AtomFeedStorage_1.AtomFeedStorage.BUCKET_NAME,
|
|
288
|
+
MessageStorage_1.MessageStorage.BUCKET_NAME,
|
|
289
|
+
ResourceStorage_1.ResourceStorage.BUCKET_NAME,
|
|
290
|
+
];
|
|
291
|
+
await Promise.all([
|
|
292
|
+
...buckets.map((name) => ensureBucket(name)),
|
|
293
|
+
ensureReleasesTable(dynamodb),
|
|
294
|
+
ensureDealsTable(dynamodb),
|
|
295
|
+
ensureUnprocessedMessagesTable(dynamodb),
|
|
296
|
+
]);
|
|
297
|
+
console.log(`Endpoint: ${ENDPOINT}`);
|
|
298
|
+
console.log(`Region: ${REGION}\n`);
|
|
299
|
+
}
|
|
300
|
+
catch (error) {
|
|
301
|
+
console.error("\n✗ Setup failed:", error instanceof Error ? error.message : error);
|
|
302
|
+
throw error;
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
exports.setupLocalStack = setupLocalStack;
|
|
306
|
+
if (require.main === module) {
|
|
307
|
+
setupLocalStack()
|
|
308
|
+
.then(() => process.exit(0))
|
|
309
|
+
.catch(() => process.exit(1));
|
|
310
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { TokenManager } from "./TokenManager";
|
|
2
|
+
import { AtomFeedStorage } from "../storage/s3/AtomFeedStorage";
|
|
3
|
+
import { MessageStorage } from "../storage/s3/MessageStorage";
|
|
4
|
+
import { ReleaseStorage } from "../storage/dynamoDb/ReleaseStorage";
|
|
5
|
+
import { DealStorage } from "../storage/dynamoDb/DealStorage";
|
|
6
|
+
import { ResourceStorage } from "../storage/s3/ResourceStorage";
|
|
7
|
+
import { UnprocessedMessageStorage } from "../storage/dynamoDb/UnprocessedMessageStorage";
|
|
8
|
+
export interface ErnManagerConfig {
|
|
9
|
+
baseUrl: string;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Manages ATOM feed and ERN message lifecycle
|
|
13
|
+
* - Fetches ATOM feed with robust error handling
|
|
14
|
+
* - Downloads ERN messages
|
|
15
|
+
* - Sends acknowledgements
|
|
16
|
+
* - Comprehensive XML parsing
|
|
17
|
+
*/
|
|
18
|
+
export declare class ErnManager {
|
|
19
|
+
private readonly tokenManager;
|
|
20
|
+
private readonly atomFeedStorage;
|
|
21
|
+
private readonly messageStorage;
|
|
22
|
+
private readonly resourceStorage;
|
|
23
|
+
private readonly releaseStorage;
|
|
24
|
+
private readonly dealStorage;
|
|
25
|
+
private readonly unprocessedMessageStorage;
|
|
26
|
+
private readonly httpClient;
|
|
27
|
+
private readonly ernParser;
|
|
28
|
+
private nextExecution;
|
|
29
|
+
private readonly endpoint;
|
|
30
|
+
constructor(config: ErnManagerConfig, tokenManager: TokenManager, atomFeedStorage: AtomFeedStorage, messageStorage: MessageStorage, resourceStorage: ResourceStorage, releaseStorage: ReleaseStorage, dealStorage: DealStorage, unprocessedMessageStorage: UnprocessedMessageStorage);
|
|
31
|
+
start(): Promise<void>;
|
|
32
|
+
getFeed(id: any): Promise<string>;
|
|
33
|
+
private getUnprocessedMessages;
|
|
34
|
+
/**
|
|
35
|
+
* Fetch ATOM feed and return entries
|
|
36
|
+
*/
|
|
37
|
+
private fetchAtomFeed;
|
|
38
|
+
/**
|
|
39
|
+
* Fetch the Ern message from the Atom feed.
|
|
40
|
+
*/
|
|
41
|
+
private fetchErnMessage;
|
|
42
|
+
private processMessage;
|
|
43
|
+
/**
|
|
44
|
+
* Send acknowledgement for individual ERN message
|
|
45
|
+
*
|
|
46
|
+
* Per SME_04_ATOM_WS_Delivery.pdf Page 11, Section 4.2.1:
|
|
47
|
+
* - POST to the "Status" URL from the ATOM entry
|
|
48
|
+
* - Use ReleaseStatus: SuccessfullyIngestedByReleaseDistributor (success)
|
|
49
|
+
* - Use ReleaseStatus: ProcessingErrorAtReleaseDistributor (failure)
|
|
50
|
+
*/
|
|
51
|
+
private acknowledgeMessage;
|
|
52
|
+
/**
|
|
53
|
+
* Build AcknowledgementMessage XML per DDEX ERN Choreography spec
|
|
54
|
+
*
|
|
55
|
+
* Per SME_04_ATOM_WS_Delivery.pdf Page 13-15, Section 5.3
|
|
56
|
+
*/
|
|
57
|
+
private buildAcknowledgementMessage;
|
|
58
|
+
}
|
|
59
|
+
//# sourceMappingURL=ErnManager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ErnManager.d.ts","sourceRoot":"","sources":["../../src/core/ErnManager.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AAEpE,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAEhE,OAAO,EAEL,yBAAyB,EAC1B,MAAM,+CAA+C,CAAC;AAOvD,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;GAMG;AACH,qBAAa,UAAU;IASnB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,cAAc;IAC/B,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,cAAc;IAC/B,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,yBAAyB;IAd5C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA8C;IACzE,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,OAAO,CAAC,aAAa,CAAK;IAE1B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;gBAGhC,MAAM,EAAE,gBAAgB,EACP,YAAY,EAAE,YAAY,EAC1B,eAAe,EAAE,eAAe,EAChC,cAAc,EAAE,cAAc,EAC9B,eAAe,EAAE,eAAe,EAChC,cAAc,EAAE,cAAc,EAC9B,WAAW,EAAE,WAAW,EACxB,yBAAyB,EAAE,yBAAyB;IAKjE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAgDtB,OAAO,CAAC,EAAE,KAAA;YAIF,sBAAsB;IAWpC;;OAEG;YACW,aAAa;IAsB3B;;OAEG;YACW,eAAe;YAiCf,cAAc;IAqB5B;;;;;;;OAOG;YACW,kBAAkB;IAqChC;;;;OAIG;IACH,OAAO,CAAC,2BAA2B;CAyDpC"}
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ErnManager = void 0;
|
|
4
|
+
const BackoffHttpClient_1 = require("../http/BackoffHttpClient");
|
|
5
|
+
const ErnParser_1 = require("./ErnParser");
|
|
6
|
+
const constants_1 = require("../config/constants");
|
|
7
|
+
const message_1 = require("../utils/message/message");
|
|
8
|
+
const SECOND = 1000;
|
|
9
|
+
const MINUTE = 60 * SECOND;
|
|
10
|
+
const HOUR = 60 * MINUTE;
|
|
11
|
+
/**
|
|
12
|
+
* Manages ATOM feed and ERN message lifecycle
|
|
13
|
+
* - Fetches ATOM feed with robust error handling
|
|
14
|
+
* - Downloads ERN messages
|
|
15
|
+
* - Sends acknowledgements
|
|
16
|
+
* - Comprehensive XML parsing
|
|
17
|
+
*/
|
|
18
|
+
class ErnManager {
|
|
19
|
+
constructor(config, tokenManager, atomFeedStorage, messageStorage, resourceStorage, releaseStorage, dealStorage, unprocessedMessageStorage) {
|
|
20
|
+
this.tokenManager = tokenManager;
|
|
21
|
+
this.atomFeedStorage = atomFeedStorage;
|
|
22
|
+
this.messageStorage = messageStorage;
|
|
23
|
+
this.resourceStorage = resourceStorage;
|
|
24
|
+
this.releaseStorage = releaseStorage;
|
|
25
|
+
this.dealStorage = dealStorage;
|
|
26
|
+
this.unprocessedMessageStorage = unprocessedMessageStorage;
|
|
27
|
+
this.httpClient = new BackoffHttpClient_1.BackoffHttpClient();
|
|
28
|
+
this.ernParser = new ErnParser_1.ErnParser();
|
|
29
|
+
this.nextExecution = 0;
|
|
30
|
+
this.endpoint = `${config.baseUrl}/gateway/ddex/ern/atom/feed/sort/date/${constants_1.VOLLEY_PARTY_ID}`;
|
|
31
|
+
}
|
|
32
|
+
async start() {
|
|
33
|
+
try {
|
|
34
|
+
console.log("Getting unprocessed messages");
|
|
35
|
+
const unprocessedMsgs = await this.getUnprocessedMessages();
|
|
36
|
+
console.log(`Processing ${unprocessedMsgs.length} unprocessed messages`);
|
|
37
|
+
for (const msg of unprocessedMsgs) {
|
|
38
|
+
await this.processMessage(msg);
|
|
39
|
+
}
|
|
40
|
+
console.log("Fetching feed");
|
|
41
|
+
const atomFeed = await this.fetchAtomFeed();
|
|
42
|
+
const hasAtomFeedMessages = atomFeed?.entries?.length > 0;
|
|
43
|
+
if (hasAtomFeedMessages) {
|
|
44
|
+
console.log(`Processing ${atomFeed.entries.length} messages`);
|
|
45
|
+
for (const entry of atomFeed.entries) {
|
|
46
|
+
const ernMsg = await this.fetchErnMessage(entry);
|
|
47
|
+
if (ernMsg) {
|
|
48
|
+
await this.processMessage(ernMsg);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
this.nextExecution = 1 * MINUTE;
|
|
52
|
+
}
|
|
53
|
+
else {
|
|
54
|
+
console.log("No entries in feed");
|
|
55
|
+
this.nextExecution = 15 * MINUTE;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
catch (error) {
|
|
59
|
+
console.log("Failed to fetch ATOM feed", error);
|
|
60
|
+
// Error - exponential backoff, max 3 hours
|
|
61
|
+
const newDelay = this.nextExecution > 0
|
|
62
|
+
? Math.min(this.nextExecution ** 2 / SECOND, 3 * HOUR)
|
|
63
|
+
: 5 * SECOND;
|
|
64
|
+
this.nextExecution = newDelay;
|
|
65
|
+
}
|
|
66
|
+
finally {
|
|
67
|
+
// Always schedule next execution
|
|
68
|
+
const nextFetchTime = Date.now() + this.nextExecution;
|
|
69
|
+
console.log(`Next fetch at ${new Date(nextFetchTime).toISOString()} (${this.nextExecution / 1000}s)`);
|
|
70
|
+
setTimeout(() => this.start(), this.nextExecution);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
async getFeed(id) {
|
|
74
|
+
return await this.atomFeedStorage.getFeed(id);
|
|
75
|
+
}
|
|
76
|
+
async getUnprocessedMessages() {
|
|
77
|
+
const unprocessedMetadata = await this.unprocessedMessageStorage.getUnprocessedMessageMetadata();
|
|
78
|
+
return await Promise.all(unprocessedMetadata.map(async (md) => this.messageStorage.getMessage(md.uuid)));
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Fetch ATOM feed and return entries
|
|
82
|
+
*/
|
|
83
|
+
async fetchAtomFeed() {
|
|
84
|
+
const token = await this.tokenManager.getToken();
|
|
85
|
+
const xmlString = await this.httpClient.get(this.endpoint, {
|
|
86
|
+
headers: {
|
|
87
|
+
Authorization: `Bearer ${token}`,
|
|
88
|
+
},
|
|
89
|
+
});
|
|
90
|
+
// Parse ATOM feed
|
|
91
|
+
const atomFeed = this.ernParser.parseAtomFeed(xmlString);
|
|
92
|
+
if (atomFeed.entries?.length > 0) {
|
|
93
|
+
// Save raw feed to Wasabi
|
|
94
|
+
await this.atomFeedStorage.saveFeed(atomFeed.id, xmlString);
|
|
95
|
+
}
|
|
96
|
+
else {
|
|
97
|
+
// TODO: logs and metrics
|
|
98
|
+
console.log("No entries");
|
|
99
|
+
}
|
|
100
|
+
return atomFeed;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Fetch the Ern message from the Atom feed.
|
|
104
|
+
*/
|
|
105
|
+
async fetchErnMessage(entry) {
|
|
106
|
+
let ernMessage;
|
|
107
|
+
try {
|
|
108
|
+
const token = await this.tokenManager.getToken();
|
|
109
|
+
const xmlString = await this.httpClient.get(entry.releaseEndpoint, {
|
|
110
|
+
headers: {
|
|
111
|
+
Authorization: `Bearer ${token}`,
|
|
112
|
+
},
|
|
113
|
+
});
|
|
114
|
+
try {
|
|
115
|
+
ernMessage = this.ernParser.parseErnMessage(xmlString);
|
|
116
|
+
await this.messageStorage.saveMessage(ernMessage);
|
|
117
|
+
await this.unprocessedMessageStorage.saveUnprocessedMessage(ernMessage);
|
|
118
|
+
await this.acknowledgeMessage(entry, true, null);
|
|
119
|
+
}
|
|
120
|
+
catch (error) {
|
|
121
|
+
console.log("Failed to parse or save ERN message:", error);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
catch (error) {
|
|
125
|
+
console.log("Failed to fetch ERN message:", error);
|
|
126
|
+
await this.acknowledgeMessage(entry, false, `${error}`);
|
|
127
|
+
}
|
|
128
|
+
return ernMessage;
|
|
129
|
+
}
|
|
130
|
+
async processMessage(ernMessage) {
|
|
131
|
+
try {
|
|
132
|
+
const messageId = (0, message_1.extractMessageId)(ernMessage);
|
|
133
|
+
console.log(`Saving ${messageId}`);
|
|
134
|
+
await Promise.all([
|
|
135
|
+
// Save resources to Wasabi S3
|
|
136
|
+
this.resourceStorage.saveResources(ernMessage),
|
|
137
|
+
// Save release metadata to DynamoDB
|
|
138
|
+
this.releaseStorage.saveReleases(ernMessage),
|
|
139
|
+
// Save deals
|
|
140
|
+
this.dealStorage.saveDeals(ernMessage),
|
|
141
|
+
]);
|
|
142
|
+
console.log(`Finished saving ${messageId}`);
|
|
143
|
+
await this.unprocessedMessageStorage.markMessageProcessed(ernMessage);
|
|
144
|
+
console.log(`Marked processed ${messageId}`);
|
|
145
|
+
}
|
|
146
|
+
catch (error) {
|
|
147
|
+
// TODO: log this
|
|
148
|
+
console.log("Processing message error: ", error);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Send acknowledgement for individual ERN message
|
|
153
|
+
*
|
|
154
|
+
* Per SME_04_ATOM_WS_Delivery.pdf Page 11, Section 4.2.1:
|
|
155
|
+
* - POST to the "Status" URL from the ATOM entry
|
|
156
|
+
* - Use ReleaseStatus: SuccessfullyIngestedByReleaseDistributor (success)
|
|
157
|
+
* - Use ReleaseStatus: ProcessingErrorAtReleaseDistributor (failure)
|
|
158
|
+
*/
|
|
159
|
+
async acknowledgeMessage(entry, success, errorText) {
|
|
160
|
+
if (!entry.ackEndpoint) {
|
|
161
|
+
console.log("No ack endpoint for entry:", entry.id);
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
const token = await this.tokenManager.getToken();
|
|
165
|
+
const grid = entry.GRid;
|
|
166
|
+
const originalMessageId = entry.id;
|
|
167
|
+
const releaseStatus = success
|
|
168
|
+
? "SuccessfullyIngestedByReleaseDistributor"
|
|
169
|
+
: "ProcessingErrorAtReleaseDistributor";
|
|
170
|
+
const ackMessage = this.buildAcknowledgementMessage(grid, originalMessageId, releaseStatus, errorText);
|
|
171
|
+
try {
|
|
172
|
+
await this.httpClient.post(entry.ackEndpoint, ackMessage, {
|
|
173
|
+
headers: {
|
|
174
|
+
Authorization: `Bearer ${token}`,
|
|
175
|
+
"Content-Type": "application/xml",
|
|
176
|
+
},
|
|
177
|
+
});
|
|
178
|
+
}
|
|
179
|
+
catch (error) {
|
|
180
|
+
console.log(`Failed to acknowledge message ${originalMessageId}:`, error);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Build AcknowledgementMessage XML per DDEX ERN Choreography spec
|
|
185
|
+
*
|
|
186
|
+
* Per SME_04_ATOM_WS_Delivery.pdf Page 13-15, Section 5.3
|
|
187
|
+
*/
|
|
188
|
+
buildAcknowledgementMessage(grid, originalMessageId, releaseStatus, errorText) {
|
|
189
|
+
const now = new Date().toISOString();
|
|
190
|
+
const ackMessageId = `${Date.now()}`;
|
|
191
|
+
let xml = `<?xml version="1.0"?>
|
|
192
|
+
<ns3:AcknowledgementMessage xmlns:xs="http://www.w3.org/2001/XMLSchema-instance"
|
|
193
|
+
xmlns:ns3="http://ddex.net/xml/ern-c-sftp/18"
|
|
194
|
+
AvsVersionId="4">
|
|
195
|
+
<MessageHeader>
|
|
196
|
+
<MessageId>${ackMessageId}</MessageId>
|
|
197
|
+
<MessageSender>
|
|
198
|
+
<PartyId>${constants_1.VOLLEY_PARTY_ID}</PartyId>
|
|
199
|
+
<PartyName>
|
|
200
|
+
<FullName>Volley</FullName>
|
|
201
|
+
</PartyName>
|
|
202
|
+
</MessageSender>
|
|
203
|
+
<MessageRecipient>
|
|
204
|
+
<PartyId>PADPIDA2007040502I</PartyId>
|
|
205
|
+
<PartyName>
|
|
206
|
+
<FullName>Sony Music Entertainment</FullName>
|
|
207
|
+
</PartyName>
|
|
208
|
+
</MessageRecipient>
|
|
209
|
+
<MessageCreatedDateTime>${now}</MessageCreatedDateTime>
|
|
210
|
+
</MessageHeader>
|
|
211
|
+
<ReleaseStatus>
|
|
212
|
+
<ReleaseId>
|
|
213
|
+
<GRid>${grid}</GRid>
|
|
214
|
+
</ReleaseId>
|
|
215
|
+
<ReleaseStatus>${releaseStatus}</ReleaseStatus>`;
|
|
216
|
+
if (errorText) {
|
|
217
|
+
xml += `
|
|
218
|
+
<ErrorText>${errorText}</ErrorText>`;
|
|
219
|
+
}
|
|
220
|
+
xml += `
|
|
221
|
+
<Acknowledgement>
|
|
222
|
+
<MessageType>NewReleaseMessage</MessageType>
|
|
223
|
+
<MessageId>${originalMessageId}</MessageId>
|
|
224
|
+
<MessageStatus>
|
|
225
|
+
<Status>${releaseStatus === "SuccessfullyIngestedByReleaseDistributor"
|
|
226
|
+
? "FileOK"
|
|
227
|
+
: "ProcessingError"}</Status>
|
|
228
|
+
</MessageStatus>
|
|
229
|
+
</Acknowledgement>
|
|
230
|
+
</ReleaseStatus>
|
|
231
|
+
</ns3:AcknowledgementMessage>`;
|
|
232
|
+
return xml;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
exports.ErnManager = ErnManager;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ErnManager.spec.d.ts","sourceRoot":"","sources":["../../src/core/ErnManager.spec.ts"],"names":[],"mappings":""}
|