@certik/skynet 0.7.17 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -1
- package/app.js +476 -0
- package/cli.js +51 -0
- package/deploy.js +188 -143
- package/examples/consumer +53 -0
- package/examples/indexer +39 -8
- package/examples/{deploy-consumer → legacy-deploy-consumer} +2 -2
- package/examples/{deploy-indexer → legacy-deploy-indexer} +4 -3
- package/examples/{deploy-mode-indexer → legacy-deploy-mode-indexer} +2 -2
- package/examples/{deploy-producer → legacy-deploy-producer} +2 -2
- package/examples/legacy-indexer +25 -0
- package/examples/{kafka-consumer → legacy-kafka-consumer} +1 -1
- package/examples/{kafka-producer → legacy-kafka-producer} +2 -2
- package/examples/legacy-mode-indexer +64 -0
- package/examples/mode-indexer +51 -36
- package/examples/producer +68 -0
- package/indexer.js +116 -57
- package/kafka.js +9 -7
- package/monitor.js +246 -0
- package/package.json +1 -1
- package/s3.js +30 -1
- package/selector.js +5 -5
- package/slack.js +47 -31
- package/token.js +9 -7
- package/package-lock.json +0 -14109
package/s3.js
CHANGED
|
@@ -84,10 +84,39 @@ async function deleteFile(bucketName, key, verbose = false) {
|
|
|
84
84
|
}
|
|
85
85
|
}
|
|
86
86
|
|
|
87
|
+
async function listKeys(bucketname, prefix, continuationToken) {
|
|
88
|
+
const s3 = getS3();
|
|
89
|
+
|
|
90
|
+
const params = {
|
|
91
|
+
Bucket: bucketname,
|
|
92
|
+
Prefix: prefix,
|
|
93
|
+
ContinuationToken: continuationToken,
|
|
94
|
+
};
|
|
95
|
+
|
|
96
|
+
let data = null;
|
|
97
|
+
try {
|
|
98
|
+
data = await s3.listObjectsV2(params).promise();
|
|
99
|
+
} catch (err) {
|
|
100
|
+
if (err.statusCode && err.statusCode === 400) {
|
|
101
|
+
return null;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
throw `unable to list keys with prefix ${prefix}: ${err}`;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
let res = { keys: data.Contents.map(({ Key }) => Key) };
|
|
108
|
+
if (data.IsTruncated) {
|
|
109
|
+
res.continuationToken = data.NextContinuationToken;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
return res;
|
|
113
|
+
}
|
|
114
|
+
|
|
87
115
|
module.exports = {
|
|
88
116
|
getS3,
|
|
89
117
|
hasFile,
|
|
90
118
|
readFile,
|
|
91
119
|
writeFile,
|
|
92
|
-
deleteFile
|
|
120
|
+
deleteFile,
|
|
121
|
+
listKeys
|
|
93
122
|
};
|
package/selector.js
CHANGED
|
@@ -9,9 +9,9 @@ function getSpaces(num) {
|
|
|
9
9
|
|
|
10
10
|
function getSelectorDesc(selector) {
|
|
11
11
|
return Object.keys(selector)
|
|
12
|
-
.map(name => {
|
|
12
|
+
.map((name) => {
|
|
13
13
|
return ` --${name}${getSpaces(14 - name.length)}${
|
|
14
|
-
selector[name].desc
|
|
14
|
+
selector[name].desc || selector[name].description || ""
|
|
15
15
|
}`;
|
|
16
16
|
})
|
|
17
17
|
.join("\n");
|
|
@@ -21,7 +21,7 @@ function getSelectorFlags(selector) {
|
|
|
21
21
|
return Object.keys(selector).reduce((acc, name) => {
|
|
22
22
|
const flag = {
|
|
23
23
|
type: selector[name].type || "string",
|
|
24
|
-
isRequired: true
|
|
24
|
+
isRequired: true,
|
|
25
25
|
};
|
|
26
26
|
|
|
27
27
|
if (selector[name].default) {
|
|
@@ -40,7 +40,7 @@ function getSelectorFlags(selector) {
|
|
|
40
40
|
function toSelectorString(selectorFlags, delim = ",") {
|
|
41
41
|
return Object.keys(selectorFlags)
|
|
42
42
|
.sort() // deterministic
|
|
43
|
-
.map(flag => {
|
|
43
|
+
.map((flag) => {
|
|
44
44
|
return `${flag}=${selectorFlags[flag]}`;
|
|
45
45
|
})
|
|
46
46
|
.join(delim);
|
|
@@ -49,5 +49,5 @@ function toSelectorString(selectorFlags, delim = ",") {
|
|
|
49
49
|
module.exports = {
|
|
50
50
|
getSelectorDesc,
|
|
51
51
|
getSelectorFlags,
|
|
52
|
-
toSelectorString
|
|
52
|
+
toSelectorString,
|
|
53
53
|
};
|
package/slack.js
CHANGED
|
@@ -1,46 +1,62 @@
|
|
|
1
|
-
const { WebClient
|
|
1
|
+
const { WebClient } = require("@slack/web-api");
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
function getToken() {
|
|
4
|
+
return process.env.SKYNET_SLACK_TOKEN;
|
|
5
|
+
}
|
|
4
6
|
|
|
5
|
-
|
|
6
|
-
const
|
|
7
|
-
logLevel: LogLevel.DEBUG,
|
|
8
|
-
});
|
|
7
|
+
function getClient() {
|
|
8
|
+
const token = getToken();
|
|
9
9
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
10
|
+
if (!token) {
|
|
11
|
+
throw new Error("Cannot communicate with slack due to missing slack token: SKYNET_SLACK_TOKEN");
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
const client = new WebClient(token);
|
|
15
|
+
|
|
16
|
+
return client;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
async function findConversation(client, name) {
|
|
20
|
+
const { conversations } = client;
|
|
21
|
+
|
|
22
|
+
// Call the conversations.list method using the built-in WebClient
|
|
23
|
+
let result = await conversations.list({
|
|
24
|
+
limit: 1000,
|
|
25
|
+
});
|
|
17
26
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
27
|
+
for (const channel of result.channels) {
|
|
28
|
+
if (channel.name === name) {
|
|
29
|
+
const conversationId = channel.id;
|
|
21
30
|
|
|
22
|
-
|
|
23
|
-
return conversationId;
|
|
24
|
-
}
|
|
31
|
+
return conversationId;
|
|
25
32
|
}
|
|
26
|
-
} catch (error) {
|
|
27
|
-
console.error(error);
|
|
28
33
|
}
|
|
34
|
+
|
|
35
|
+
return null;
|
|
29
36
|
}
|
|
30
37
|
|
|
31
|
-
async function
|
|
38
|
+
async function postMessage(channel, message) {
|
|
32
39
|
try {
|
|
33
|
-
const
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
text: text,
|
|
37
|
-
// You could also use a blocks[] array to send richer content
|
|
38
|
-
});
|
|
40
|
+
const client = getClient();
|
|
41
|
+
|
|
42
|
+
const conversationId = await findConversation(client, channel);
|
|
39
43
|
|
|
40
|
-
|
|
44
|
+
let post = {};
|
|
45
|
+
|
|
46
|
+
if (typeof message === "string") {
|
|
47
|
+
post.text = message;
|
|
48
|
+
} else {
|
|
49
|
+
post = message;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
await client.chat.postMessage({
|
|
53
|
+
channel: conversationId,
|
|
54
|
+
...post,
|
|
55
|
+
});
|
|
41
56
|
} catch (error) {
|
|
42
|
-
|
|
57
|
+
// no blocking
|
|
58
|
+
console.error("failed to post slack message", error);
|
|
43
59
|
}
|
|
44
60
|
}
|
|
45
61
|
|
|
46
|
-
module.exports = {
|
|
62
|
+
module.exports = { postMessage };
|
package/token.js
CHANGED
|
@@ -5,24 +5,26 @@ const BigNumber = require("bignumber.js");
|
|
|
5
5
|
const TOKEN_PRICE_TABLE_NAME = `skynet-${getEnvironment()}-token-prices`;
|
|
6
6
|
const TOKEN_PRICE_CACHE = {};
|
|
7
7
|
|
|
8
|
-
async function getTokenPriceAt(tokenAddress, timestamp) {
|
|
8
|
+
async function getTokenPriceAt(tokenAddress, timestamp, useCache = true) {
|
|
9
9
|
// to avoid huge amount of dynamodb query
|
|
10
10
|
// "ceil" timestamp to the closest proximate timestamp
|
|
11
11
|
// 100_000 seconds ~= a bit more than 1 day 86_400 seconds
|
|
12
12
|
// and use cache
|
|
13
13
|
const proximateTimestamp = timestamp + 100_000 - (timestamp % 100_000);
|
|
14
14
|
|
|
15
|
-
if (TOKEN_PRICE_CACHE[tokenAddress] && TOKEN_PRICE_CACHE[tokenAddress][proximateTimestamp]) {
|
|
15
|
+
if (useCache && TOKEN_PRICE_CACHE[tokenAddress] && TOKEN_PRICE_CACHE[tokenAddress][proximateTimestamp]) {
|
|
16
16
|
return TOKEN_PRICE_CACHE[tokenAddress][proximateTimestamp];
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
if (!TOKEN_PRICE_CACHE[tokenAddress]) {
|
|
20
|
-
TOKEN_PRICE_CACHE[tokenAddress] = {};
|
|
21
|
-
}
|
|
22
|
-
|
|
23
19
|
const price = await getPriceClosestTo(TOKEN_PRICE_TABLE_NAME, tokenAddress, proximateTimestamp);
|
|
24
20
|
|
|
25
|
-
|
|
21
|
+
if (useCache) {
|
|
22
|
+
if (!TOKEN_PRICE_CACHE[tokenAddress]) {
|
|
23
|
+
TOKEN_PRICE_CACHE[tokenAddress] = {};
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
TOKEN_PRICE_CACHE[tokenAddress][proximateTimestamp] = price;
|
|
27
|
+
}
|
|
26
28
|
|
|
27
29
|
return price;
|
|
28
30
|
}
|