@dcl/opscli 1.0.0-4377443452.commit-5f489bc → 1.0.0-4886526709.commit-ee4aa6b
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue-ab-conversion-snapshot.d.ts","sourceRoot":"","sources":["../../src/commands/queue-ab-conversion-snapshot.ts"],"names":[],"mappings":";
|
|
1
|
+
{"version":3,"file":"queue-ab-conversion-snapshot.d.ts","sourceRoot":"","sources":["../../src/commands/queue-ab-conversion-snapshot.ts"],"names":[],"mappings":";AAUA,wBAqGC"}
|
|
@@ -9,6 +9,7 @@ const undici_1 = require("undici");
|
|
|
9
9
|
const bin_1 = require("../bin");
|
|
10
10
|
const assert_1 = require("../helpers/assert");
|
|
11
11
|
const asset_bundles_1 = require("../helpers/asset-bundles");
|
|
12
|
+
const string_decoder_1 = require("string_decoder");
|
|
12
13
|
// PROCESS AN ENTIRE SNAPSHOT
|
|
13
14
|
exports.default = async () => {
|
|
14
15
|
const args = (0, arg_1.default)({
|
|
@@ -31,50 +32,101 @@ exports.default = async () => {
|
|
|
31
32
|
console.log(` Content server: ${contentUrl}`);
|
|
32
33
|
console.log(` Asset bundle server: ${abServer}`);
|
|
33
34
|
console.log(`> Fetching snapshots`);
|
|
34
|
-
const snapshotReq = await (0, undici_1.fetch)(`${contentUrl}/
|
|
35
|
+
const snapshotReq = await (0, undici_1.fetch)(`${contentUrl}/snapshots`);
|
|
35
36
|
if (!snapshotReq.ok)
|
|
36
|
-
throw new bin_1.CliError(`Invalid snapshot response from ${contentUrl}/
|
|
37
|
+
throw new bin_1.CliError(`Invalid snapshot response from ${contentUrl}/snapshots`);
|
|
37
38
|
const snapshotsJson = await snapshotReq.json();
|
|
38
|
-
if (!(snapshot in snapshotsJson.entities))
|
|
39
|
-
throw new bin_1.CliError(`Invalid snapshot ${snapshot}`);
|
|
40
|
-
const hash = snapshotsJson.entities[snapshot].hash;
|
|
41
|
-
console.log(`> Fetching file ${hash}`);
|
|
42
|
-
const jsonNdReq = await (0, undici_1.fetch)(`${contentUrl}/contents/${hash}`);
|
|
43
|
-
if (!jsonNdReq.ok)
|
|
44
|
-
throw new bin_1.CliError(`Invalid response from ${contentUrl}/contents/${hash}`);
|
|
45
|
-
const jsonNd = await jsonNdReq.text();
|
|
46
|
-
const len = jsonNd.length;
|
|
47
|
-
console.log(` File length ${(len / 1024 / 1024).toFixed(1)}MB`);
|
|
48
|
-
let currentCursor = (args['--start-position'] ? jsonNd.indexOf(args['--start-position']) : 0) || -1;
|
|
49
39
|
const startDate = (args['--start-date'] ? new Date(args['--start-date']).getTime() : 0) || -1;
|
|
50
|
-
let
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
40
|
+
let snapshotEntitiesToProcess = Array();
|
|
41
|
+
// select valid snapshots based on the start date
|
|
42
|
+
for (let i = 0; i < snapshotsJson.length; i++) {
|
|
43
|
+
const value = snapshotsJson[i];
|
|
44
|
+
const toTime = value.timeRange.endTimestamp;
|
|
45
|
+
if (toTime >= startDate) {
|
|
46
|
+
snapshotEntitiesToProcess.push(value);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
const snapshotsCount = snapshotEntitiesToProcess.length;
|
|
50
|
+
if (snapshotsCount == 0) {
|
|
51
|
+
throw new bin_1.CliError(`No snapshot entity found at time ${startDate}`);
|
|
52
|
+
}
|
|
53
|
+
let argStartPosition = args['--start-position'];
|
|
54
|
+
let argGrep = args['--grep'];
|
|
55
|
+
let shouldSkipUntilStartPosition = argStartPosition || false;
|
|
56
|
+
for (let i = 0; i < snapshotsCount; i++) {
|
|
57
|
+
const hash = snapshotEntitiesToProcess[i].hash;
|
|
58
|
+
const numberOfEntities = snapshotEntitiesToProcess[i].numberOfEntities;
|
|
59
|
+
console.log(`> (${i + 1}/${snapshotsCount}) Fetching file ${hash} with ${numberOfEntities} entities`);
|
|
60
|
+
const snapshotUrl = `${contentUrl}/contents/${hash}`;
|
|
61
|
+
await processSnapshot(snapshotUrl, async (line, index) => {
|
|
62
|
+
const percent = (100 * (index / numberOfEntities)).toFixed(2);
|
|
63
|
+
try {
|
|
64
|
+
if (argGrep && !line.match(argGrep))
|
|
65
|
+
return;
|
|
66
|
+
const entity = JSON.parse(line);
|
|
67
|
+
if (shouldSkipUntilStartPosition) {
|
|
68
|
+
if (entity.entityId == argStartPosition) {
|
|
69
|
+
shouldSkipUntilStartPosition = false;
|
|
70
|
+
}
|
|
71
|
+
else {
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
if (startDate <= entity.entityTimestamp && entity.entityType == snapshot) {
|
|
76
|
+
await (0, asset_bundles_1.queueConversion)(abServer, {
|
|
77
|
+
entity: {
|
|
78
|
+
entityId: entity.entityId, authChain: [
|
|
79
|
+
{
|
|
80
|
+
type: schemas_1.AuthLinkType.SIGNER,
|
|
81
|
+
payload: '0x0000000000000000000000000000000000000000',
|
|
82
|
+
signature: ''
|
|
83
|
+
}
|
|
84
|
+
]
|
|
85
|
+
}, contentServerUrls: [contentUrl]
|
|
86
|
+
}, token);
|
|
87
|
+
console.log(` (${i + 1}/${snapshotsCount}) [${percent}%]`, entity.entityId, entity.pointers[0]);
|
|
59
88
|
}
|
|
60
89
|
}
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
await (0, asset_bundles_1.queueConversion)(abServer, {
|
|
64
|
-
entity: {
|
|
65
|
-
entityId: entity.entityId, authChain: [
|
|
66
|
-
{
|
|
67
|
-
type: schemas_1.AuthLinkType.SIGNER,
|
|
68
|
-
payload: '0x0000000000000000000000000000000000000000',
|
|
69
|
-
signature: ''
|
|
70
|
-
}
|
|
71
|
-
]
|
|
72
|
-
}, contentServerUrls: [contentUrl]
|
|
73
|
-
}, token);
|
|
74
|
-
console.log(`[${percent}%]`, entity.entityId, entity.pointers[0]);
|
|
90
|
+
catch (error) {
|
|
91
|
+
console.log(error);
|
|
75
92
|
}
|
|
76
|
-
}
|
|
93
|
+
});
|
|
77
94
|
}
|
|
78
95
|
console.log(`Finished!`);
|
|
79
96
|
};
|
|
97
|
+
const processSnapshot = async (url, processLine) => {
|
|
98
|
+
const decoder = new string_decoder_1.StringDecoder('utf8');
|
|
99
|
+
let remaining = '';
|
|
100
|
+
try {
|
|
101
|
+
const response = await (0, undici_1.fetch)(url);
|
|
102
|
+
if (!response.ok)
|
|
103
|
+
throw new bin_1.CliError(`Invalid response from ${url}`);
|
|
104
|
+
let index = 0;
|
|
105
|
+
for await (const data of response.body) {
|
|
106
|
+
const chunk = decoder.write(data);
|
|
107
|
+
const lines = (remaining + chunk).split('\n');
|
|
108
|
+
if (!chunk.endsWith('\n')) {
|
|
109
|
+
remaining = lines.pop();
|
|
110
|
+
}
|
|
111
|
+
else {
|
|
112
|
+
remaining = '';
|
|
113
|
+
}
|
|
114
|
+
for (const line of lines) {
|
|
115
|
+
if (line.trim().startsWith('{')) {
|
|
116
|
+
await processLine(line, index);
|
|
117
|
+
}
|
|
118
|
+
index++;
|
|
119
|
+
process.stdout.write(index + '\r');
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
if (remaining) {
|
|
123
|
+
if (remaining.trim().startsWith('{')) {
|
|
124
|
+
await processLine(remaining, index);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
catch (error) {
|
|
129
|
+
console.error(`Failed to download file: ${error}`);
|
|
130
|
+
}
|
|
131
|
+
};
|
|
80
132
|
//# sourceMappingURL=queue-ab-conversion-snapshot.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue-ab-conversion-snapshot.js","sourceRoot":"","sources":["../../src/commands/queue-ab-conversion-snapshot.ts"],"names":[],"mappings":";;;;;AAAA,0CAA2C;AAC3C,8CAAqB;AACrB,mCAA8B;AAC9B,gCAAiC;AACjC,8CAA0C;AAC1C,4DAA0D;
|
|
1
|
+
{"version":3,"file":"queue-ab-conversion-snapshot.js","sourceRoot":"","sources":["../../src/commands/queue-ab-conversion-snapshot.ts"],"names":[],"mappings":";;;;;AAAA,0CAA2C;AAC3C,8CAAqB;AACrB,mCAA8B;AAC9B,gCAAiC;AACjC,8CAA0C;AAC1C,4DAA0D;AAC1D,mDAA8C;AAE9C,6BAA6B;AAE7B,kBAAe,KAAK,IAAI,EAAE;IACxB,MAAM,IAAI,GAAG,IAAA,aAAG,EAAC;QACf,YAAY,EAAE,MAAM;QACpB,kBAAkB,EAAE,MAAM;QAC1B,kBAAkB,EAAE,MAAM;QAC1B,cAAc,EAAE,MAAM;QACtB,QAAQ,EAAE,MAAM;QAChB,aAAa,EAAE,MAAM;QACrB,SAAS,EAAE,MAAM;KAClB,CAAC,CAAA;IAEF,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,UAAU,CAAA;IACjD,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAE,CAAA;IAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,iDAAiD,CAAA;IAEzF,IAAA,eAAM,EAAC,CAAC,CAAC,QAAQ,EAAE,uBAAuB,CAAC,CAAA;IAC3C,IAAA,eAAM,EAAC,CAAC,CAAC,KAAK,EAAE,oBAAoB,CAAC,CAAA;IAErC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAA;IAC5C,MAAM,UAAU,GAAG,CAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,uCAAuC,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;IAC3G,OAAO,CAAC,GAAG,CAAC,iCAAiC,QAAQ,EAAE,CAAC,CAAA;IACxD,OAAO,CAAC,GAAG,CAAC,iCAAiC,UAAU,EAAE,CAAC,CAAA;IAC1D,OAAO,CAAC,GAAG,CAAC,iCAAiC,QAAQ,EAAE,CAAC,CAAA;IAExD,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAA;IACnC,MAAM,WAAW,GAAG,MAAM,IAAA,cAAK,EAAC,GAAG,UAAU,YAAY,CAAC,CAAA;IAC1D,IAAI,CAAC,WAAW,CAAC,EAAE;QAAE,MAAM,IAAI,cAAQ,CAAC,kCAAkC,UAAU,YAAY,CAAC,CAAA;IAEjG,MAAM,aAAa,GAAG,MAAM,WAAW,CAAC,IAAI,EAAgB,CAAA;IAC5D,MAAM,SAAS,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;IAC7F,IAAI,yBAAyB,GAAG,KAAK,EAAO,CAAA;IAE5C,iDAAiD;IACjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAC7C;QACE,MAAM,KAAK,GAAG,aAAa,CAAC,CAAC,CAAC,CAAA;QAC9B,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,YAAY,CAAA;QAE3C,IAAI,MAAM,IAAI,SAAS,EACvB;YACE,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;SACtC;KACF;IAED,MAAM,cAAc,GAAG,yBAAyB,CAAC,MAAM,CAAC;IACxD,IAAI,cAAc,IAAI,CAAC,EACvB;QACE,MAAM,IAAI,cAAQ,CAAC,oCAAoC,SAAS,EAAE,CAAC,CAAA;KACpE;IAED,IAAI,gBAAgB,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAA;IAC/C,IAAI,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAA;IAC5B,IAAI,4BAA4B,GAAG,gBAAgB,IAAI,KAAK,CAAA;IAE5D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,cAAc,EAAE,CAAC,EAAE,EACvC;QACE,MAAM,IAAI,GAAG,yBAAyB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC/C,MAAM,gBAAgB,GAAG,yBAAyB,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAA;QACtE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAC,CAAC,IAAI,cAAc,mBAAmB,IAAI,SAAS,gBAAgB,WAAW,CAAC,CAAA;QACnG,MAAM,WAAW,GAAG,GAAG,UAAU,aAAa,IAAI,EAAE,CAAA;QAEpD,MAAM,eAAe,CAAC,WAAW,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YACvD,MAAM,OAAO,GAAG,CAAC,GAAG,GAAG,CAAC,KAAK,GAAG,gBAAgB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;YAC7D,IAAI;gBACF,IAAI,OAAO,IAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;oBAAE,OAAM;gBAE1C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;gBAE/B,IAAI,4BAA4B,EAChC;oBACE,IAAI,MAAM,CAAC,QAAQ,IAAI,gBAAgB,EACvC;wBACE,4BAA4B,GAAG,KAAK,CAAA;qBACrC;yBACD;wBACE,OAAO;qBACR;iBACF;gBAED,IAAI,SAAS,IAAI,MAAM,CAAC,eAAe,IAAI,MAAM,CAAC,UAAU,IAAI,QAAQ,EAAE;oBACxE,MAAM,IAAA,+BAAe,EAAC,QAAQ,EAAE;wBAC9B,MAAM,EAAE;4BACN,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,SAAS,EAAE;gCACpC;oCACE,IAAI,EAAE,sBAAY,CAAC,MAAM;oCACzB,OAAO,EAAE,4CAA4C;oCACrD,SAAS,EAAE,EAAE;iCACd;6BACF;yBACF,EAAE,iBAAiB,EAAE,CAAC,UAAU,CAAC;qBACnC,EAAE,KAAK,CAAC,CAAA;oBAET,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAC,CAAC,IAAI,cAAc,MAAM,OAAO,IAAI,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;iBAC/F;aACF;YAAC,OAAO,KAAK,EAAE;gBACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;aACnB;QACH,CAAC,CAAC,CAAA;KACH;IAED,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;AAC1B,CAAC,CAAA;AAED,MAAM,eAAe,GAAG,KAAK,EAAE,GAAO,EAAE,WAAyD,EAAE,EAAE;IACnG,MAAM,OAAO,GAAG,IAAI,8BAAa,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,SAAS,GAAG,EAAE,CAAC;IAEnB,IAAI;QACF,MAAM,QAAQ,GAAG,MAAM,IAAA,cAAK,EAAC,GAAG,CAAC,CAAC;QAClC,IAAI,CAAC,QAAQ,CAAC,EAAE;YAAE,MAAM,IAAI,cAAQ,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAAA;QAEpE,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,QAAQ,CAAC,IAAI,EAAE;YACtC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAClC,MAAM,KAAK,GAAG,CAAC,SAAS,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAE9C,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;gBACzB,SAAS,GAAG,KAAK,CAAC,GAAG,EAAG,CAAC;aAC1B;iBAAM;gBACL,SAAS,GAAG,EAAE,CAAC;aAChB;YAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;gBACxB,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;oBAC/B,MAAM,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;iBAC/B;gBACD,KAAK,EAAE,CAAA;gBACP,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,CAAA;aACnC;SACF;QAED,IAAI,SAAS,EAAE;YACb,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;gBACpC,MAAM,WAAW,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;aACpC;SACF;KAEF;IAAC,OAAO,KAAK,EAAE;QACd,OAAO,CAAC,KAAK,CAAC,4BAA4B,KAAK,EAAE,CAAC,CAAC;KACpD;AACH,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dcl/opscli",
|
|
3
|
-
"version": "1.0.0-
|
|
3
|
+
"version": "1.0.0-4886526709.commit-ee4aa6b",
|
|
4
4
|
"description": "base component",
|
|
5
5
|
"bin": "dist/bin.js",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -44,5 +44,5 @@
|
|
|
44
44
|
"files": [
|
|
45
45
|
"dist"
|
|
46
46
|
],
|
|
47
|
-
"commit": "
|
|
47
|
+
"commit": "ee4aa6b5de1eedc300f552678969122519ef4358"
|
|
48
48
|
}
|