@dcl/opscli 1.0.0-4877206984.commit-8e8a724 → 1.0.0-4886526709.commit-ee4aa6b
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue-ab-conversion-snapshot.d.ts","sourceRoot":"","sources":["../../src/commands/queue-ab-conversion-snapshot.ts"],"names":[],"mappings":";
|
|
1
|
+
{"version":3,"file":"queue-ab-conversion-snapshot.d.ts","sourceRoot":"","sources":["../../src/commands/queue-ab-conversion-snapshot.ts"],"names":[],"mappings":";AAUA,wBAqGC"}
|
|
@@ -9,6 +9,7 @@ const undici_1 = require("undici");
|
|
|
9
9
|
const bin_1 = require("../bin");
|
|
10
10
|
const assert_1 = require("../helpers/assert");
|
|
11
11
|
const asset_bundles_1 = require("../helpers/asset-bundles");
|
|
12
|
+
const string_decoder_1 = require("string_decoder");
|
|
12
13
|
// PROCESS AN ENTIRE SNAPSHOT
|
|
13
14
|
exports.default = async () => {
|
|
14
15
|
const args = (0, arg_1.default)({
|
|
@@ -40,7 +41,6 @@ exports.default = async () => {
|
|
|
40
41
|
// select valid snapshots based on the start date
|
|
41
42
|
for (let i = 0; i < snapshotsJson.length; i++) {
|
|
42
43
|
const value = snapshotsJson[i];
|
|
43
|
-
const fromTime = value.timeRange.initTimestamp;
|
|
44
44
|
const toTime = value.timeRange.endTimestamp;
|
|
45
45
|
if (toTime >= startDate) {
|
|
46
46
|
snapshotEntitiesToProcess.push(value);
|
|
@@ -50,47 +50,30 @@ exports.default = async () => {
|
|
|
50
50
|
if (snapshotsCount == 0) {
|
|
51
51
|
throw new bin_1.CliError(`No snapshot entity found at time ${startDate}`);
|
|
52
52
|
}
|
|
53
|
-
let startPosition = -1;
|
|
54
53
|
let argStartPosition = args['--start-position'];
|
|
54
|
+
let argGrep = args['--grep'];
|
|
55
|
+
let shouldSkipUntilStartPosition = argStartPosition || false;
|
|
55
56
|
for (let i = 0; i < snapshotsCount; i++) {
|
|
56
57
|
const hash = snapshotEntitiesToProcess[i].hash;
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
}
|
|
73
|
-
else {
|
|
74
|
-
currentCursor = startPosition;
|
|
75
|
-
}
|
|
76
|
-
if (currentCursor < 0) {
|
|
77
|
-
console.log(` skipped because of --start-position`);
|
|
78
|
-
continue;
|
|
79
|
-
}
|
|
80
|
-
let nextCursor = 0;
|
|
81
|
-
while ((nextCursor = jsonNd.indexOf('\n', currentCursor + 1)) != -1) {
|
|
82
|
-
const line = jsonNd.substring(currentCursor, nextCursor);
|
|
83
|
-
currentCursor = nextCursor;
|
|
84
|
-
if (line.trim().startsWith('{')) {
|
|
85
|
-
const percent = (100 * (nextCursor / len)).toFixed(2);
|
|
86
|
-
if (args['--grep']) {
|
|
87
|
-
if (!line.match(args['--grep'])) {
|
|
88
|
-
continue;
|
|
58
|
+
const numberOfEntities = snapshotEntitiesToProcess[i].numberOfEntities;
|
|
59
|
+
console.log(`> (${i + 1}/${snapshotsCount}) Fetching file ${hash} with ${numberOfEntities} entities`);
|
|
60
|
+
const snapshotUrl = `${contentUrl}/contents/${hash}`;
|
|
61
|
+
await processSnapshot(snapshotUrl, async (line, index) => {
|
|
62
|
+
const percent = (100 * (index / numberOfEntities)).toFixed(2);
|
|
63
|
+
try {
|
|
64
|
+
if (argGrep && !line.match(argGrep))
|
|
65
|
+
return;
|
|
66
|
+
const entity = JSON.parse(line);
|
|
67
|
+
if (shouldSkipUntilStartPosition) {
|
|
68
|
+
if (entity.entityId == argStartPosition) {
|
|
69
|
+
shouldSkipUntilStartPosition = false;
|
|
70
|
+
}
|
|
71
|
+
else {
|
|
72
|
+
return;
|
|
89
73
|
}
|
|
90
74
|
}
|
|
91
|
-
const entity = JSON.parse(line);
|
|
92
75
|
if (startDate <= entity.entityTimestamp && entity.entityType == snapshot) {
|
|
93
|
-
|
|
76
|
+
await (0, asset_bundles_1.queueConversion)(abServer, {
|
|
94
77
|
entity: {
|
|
95
78
|
entityId: entity.entityId, authChain: [
|
|
96
79
|
{
|
|
@@ -104,8 +87,46 @@ exports.default = async () => {
|
|
|
104
87
|
console.log(` (${i + 1}/${snapshotsCount}) [${percent}%]`, entity.entityId, entity.pointers[0]);
|
|
105
88
|
}
|
|
106
89
|
}
|
|
107
|
-
|
|
90
|
+
catch (error) {
|
|
91
|
+
console.log(error);
|
|
92
|
+
}
|
|
93
|
+
});
|
|
108
94
|
}
|
|
109
95
|
console.log(`Finished!`);
|
|
110
96
|
};
|
|
97
|
+
const processSnapshot = async (url, processLine) => {
|
|
98
|
+
const decoder = new string_decoder_1.StringDecoder('utf8');
|
|
99
|
+
let remaining = '';
|
|
100
|
+
try {
|
|
101
|
+
const response = await (0, undici_1.fetch)(url);
|
|
102
|
+
if (!response.ok)
|
|
103
|
+
throw new bin_1.CliError(`Invalid response from ${url}`);
|
|
104
|
+
let index = 0;
|
|
105
|
+
for await (const data of response.body) {
|
|
106
|
+
const chunk = decoder.write(data);
|
|
107
|
+
const lines = (remaining + chunk).split('\n');
|
|
108
|
+
if (!chunk.endsWith('\n')) {
|
|
109
|
+
remaining = lines.pop();
|
|
110
|
+
}
|
|
111
|
+
else {
|
|
112
|
+
remaining = '';
|
|
113
|
+
}
|
|
114
|
+
for (const line of lines) {
|
|
115
|
+
if (line.trim().startsWith('{')) {
|
|
116
|
+
await processLine(line, index);
|
|
117
|
+
}
|
|
118
|
+
index++;
|
|
119
|
+
process.stdout.write(index + '\r');
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
if (remaining) {
|
|
123
|
+
if (remaining.trim().startsWith('{')) {
|
|
124
|
+
await processLine(remaining, index);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
catch (error) {
|
|
129
|
+
console.error(`Failed to download file: ${error}`);
|
|
130
|
+
}
|
|
131
|
+
};
|
|
111
132
|
//# sourceMappingURL=queue-ab-conversion-snapshot.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue-ab-conversion-snapshot.js","sourceRoot":"","sources":["../../src/commands/queue-ab-conversion-snapshot.ts"],"names":[],"mappings":";;;;;AAAA,0CAA2C;AAC3C,8CAAqB;AACrB,mCAA8B;AAC9B,gCAAiC;AACjC,8CAA0C;AAC1C,4DAA0D;
|
|
1
|
+
{"version":3,"file":"queue-ab-conversion-snapshot.js","sourceRoot":"","sources":["../../src/commands/queue-ab-conversion-snapshot.ts"],"names":[],"mappings":";;;;;AAAA,0CAA2C;AAC3C,8CAAqB;AACrB,mCAA8B;AAC9B,gCAAiC;AACjC,8CAA0C;AAC1C,4DAA0D;AAC1D,mDAA8C;AAE9C,6BAA6B;AAE7B,kBAAe,KAAK,IAAI,EAAE;IACxB,MAAM,IAAI,GAAG,IAAA,aAAG,EAAC;QACf,YAAY,EAAE,MAAM;QACpB,kBAAkB,EAAE,MAAM;QAC1B,kBAAkB,EAAE,MAAM;QAC1B,cAAc,EAAE,MAAM;QACtB,QAAQ,EAAE,MAAM;QAChB,aAAa,EAAE,MAAM;QACrB,SAAS,EAAE,MAAM;KAClB,CAAC,CAAA;IAEF,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,UAAU,CAAA;IACjD,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAE,CAAA;IAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,iDAAiD,CAAA;IAEzF,IAAA,eAAM,EAAC,CAAC,CAAC,QAAQ,EAAE,uBAAuB,CAAC,CAAA;IAC3C,IAAA,eAAM,EAAC,CAAC,CAAC,KAAK,EAAE,oBAAoB,CAAC,CAAA;IAErC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAA;IAC5C,MAAM,UAAU,GAAG,CAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,uCAAuC,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;IAC3G,OAAO,CAAC,GAAG,CAAC,iCAAiC,QAAQ,EAAE,CAAC,CAAA;IACxD,OAAO,CAAC,GAAG,CAAC,iCAAiC,UAAU,EAAE,CAAC,CAAA;IAC1D,OAAO,CAAC,GAAG,CAAC,iCAAiC,QAAQ,EAAE,CAAC,CAAA;IAExD,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAA;IACnC,MAAM,WAAW,GAAG,MAAM,IAAA,cAAK,EAAC,GAAG,UAAU,YAAY,CAAC,CAAA;IAC1D,IAAI,CAAC,WAAW,CAAC,EAAE;QAAE,MAAM,IAAI,cAAQ,CAAC,kCAAkC,UAAU,YAAY,CAAC,CAAA;IAEjG,MAAM,aAAa,GAAG,MAAM,WAAW,CAAC,IAAI,EAAgB,CAAA;IAC5D,MAAM,SAAS,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;IAC7F,IAAI,yBAAyB,GAAG,KAAK,EAAO,CAAA;IAE5C,iDAAiD;IACjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAC7C;QACE,MAAM,KAAK,GAAG,aAAa,CAAC,CAAC,CAAC,CAAA;QAC9B,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,YAAY,CAAA;QAE3C,IAAI,MAAM,IAAI,SAAS,EACvB;YACE,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;SACtC;KACF;IAED,MAAM,cAAc,GAAG,yBAAyB,CAAC,MAAM,CAAC;IACxD,IAAI,cAAc,IAAI,CAAC,EACvB;QACE,MAAM,IAAI,cAAQ,CAAC,oCAAoC,SAAS,EAAE,CAAC,CAAA;KACpE;IAED,IAAI,gBAAgB,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAA;IAC/C,IAAI,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAA;IAC5B,IAAI,4BAA4B,GAAG,gBAAgB,IAAI,KAAK,CAAA;IAE5D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,cAAc,EAAE,CAAC,EAAE,EACvC;QACE,MAAM,IAAI,GAAG,yBAAyB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC/C,MAAM,gBAAgB,GAAG,yBAAyB,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAA;QACtE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAC,CAAC,IAAI,cAAc,mBAAmB,IAAI,SAAS,gBAAgB,WAAW,CAAC,CAAA;QACnG,MAAM,WAAW,GAAG,GAAG,UAAU,aAAa,IAAI,EAAE,CAAA;QAEpD,MAAM,eAAe,CAAC,WAAW,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YACvD,MAAM,OAAO,GAAG,CAAC,GAAG,GAAG,CAAC,KAAK,GAAG,gBAAgB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;YAC7D,IAAI;gBACF,IAAI,OAAO,IAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;oBAAE,OAAM;gBAE1C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;gBAE/B,IAAI,4BAA4B,EAChC;oBACE,IAAI,MAAM,CAAC,QAAQ,IAAI,gBAAgB,EACvC;wBACE,4BAA4B,GAAG,KAAK,CAAA;qBACrC;yBACD;wBACE,OAAO;qBACR;iBACF;gBAED,IAAI,SAAS,IAAI,MAAM,CAAC,eAAe,IAAI,MAAM,CAAC,UAAU,IAAI,QAAQ,EAAE;oBACxE,MAAM,IAAA,+BAAe,EAAC,QAAQ,EAAE;wBAC9B,MAAM,EAAE;4BACN,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,SAAS,EAAE;gCACpC;oCACE,IAAI,EAAE,sBAAY,CAAC,MAAM;oCACzB,OAAO,EAAE,4CAA4C;oCACrD,SAAS,EAAE,EAAE;iCACd;6BACF;yBACF,EAAE,iBAAiB,EAAE,CAAC,UAAU,CAAC;qBACnC,EAAE,KAAK,CAAC,CAAA;oBAET,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAC,CAAC,IAAI,cAAc,MAAM,OAAO,IAAI,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;iBAC/F;aACF;YAAC,OAAO,KAAK,EAAE;gBACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;aACnB;QACH,CAAC,CAAC,CAAA;KACH;IAED,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;AAC1B,CAAC,CAAA;AAED,MAAM,eAAe,GAAG,KAAK,EAAE,GAAO,EAAE,WAAyD,EAAE,EAAE;IACnG,MAAM,OAAO,GAAG,IAAI,8BAAa,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,SAAS,GAAG,EAAE,CAAC;IAEnB,IAAI;QACF,MAAM,QAAQ,GAAG,MAAM,IAAA,cAAK,EAAC,GAAG,CAAC,CAAC;QAClC,IAAI,CAAC,QAAQ,CAAC,EAAE;YAAE,MAAM,IAAI,cAAQ,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAAA;QAEpE,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,QAAQ,CAAC,IAAI,EAAE;YACtC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAClC,MAAM,KAAK,GAAG,CAAC,SAAS,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAE9C,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;gBACzB,SAAS,GAAG,KAAK,CAAC,GAAG,EAAG,CAAC;aAC1B;iBAAM;gBACL,SAAS,GAAG,EAAE,CAAC;aAChB;YAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;gBACxB,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;oBAC/B,MAAM,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;iBAC/B;gBACD,KAAK,EAAE,CAAA;gBACP,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,CAAA;aACnC;SACF;QAED,IAAI,SAAS,EAAE;YACb,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;gBACpC,MAAM,WAAW,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;aACpC;SACF;KAEF;IAAC,OAAO,KAAK,EAAE;QACd,OAAO,CAAC,KAAK,CAAC,4BAA4B,KAAK,EAAE,CAAC,CAAC;KACpD;AACH,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dcl/opscli",
|
|
3
|
-
"version": "1.0.0-
|
|
3
|
+
"version": "1.0.0-4886526709.commit-ee4aa6b",
|
|
4
4
|
"description": "base component",
|
|
5
5
|
"bin": "dist/bin.js",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -44,5 +44,5 @@
|
|
|
44
44
|
"files": [
|
|
45
45
|
"dist"
|
|
46
46
|
],
|
|
47
|
-
"commit": "
|
|
47
|
+
"commit": "ee4aa6b5de1eedc300f552678969122519ef4358"
|
|
48
48
|
}
|