@certik/skynet 0.16.7 → 0.17.0-beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -3
- package/app.js +12 -217
- package/const.d.ts +0 -1
- package/const.js +1 -3
- package/deploy.js +18 -92
- package/examples/api.js +1 -1
- package/examples/consumer.js +2 -19
- package/examples/indexer.js +5 -27
- package/examples/mode-indexer.js +2 -21
- package/examples/producer.js +2 -19
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
-
## 0.
|
|
3
|
+
## 0.17.0
|
|
4
4
|
|
|
5
|
-
-
|
|
5
|
+
- BREAKING: changed the way an indexer integrate with added doppler integration support in deploy
|
|
6
|
+
- BREAKING: removed checks from deployment
|
|
6
7
|
|
|
7
|
-
## 0.16.6
|
|
8
|
+
## 0.16.6/0.16.7/0.16.8
|
|
8
9
|
|
|
9
10
|
- Updated: use larger cpu/mem for service monitor process
|
|
10
11
|
|
package/app.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { EOL } from "os";
|
|
2
|
-
import { SKYNET_API_PREFIX } from "./const.js";
|
|
3
2
|
|
|
4
3
|
import {
|
|
5
4
|
createIndexerApp,
|
|
@@ -15,11 +14,11 @@ import { startApiApp } from "./api.js";
|
|
|
15
14
|
import { createMonitor, ERROR_LEVEL } from "./monitor.js";
|
|
16
15
|
import { getBinaryName, detectBin, detectWorkingDirectory } from "./cli.js";
|
|
17
16
|
|
|
18
|
-
function printAppHelp(
|
|
17
|
+
function printAppHelp() {
|
|
19
18
|
console.log(`
|
|
20
19
|
Usage
|
|
21
20
|
|
|
22
|
-
$ ${getBinaryName()} run <options
|
|
21
|
+
$ ${getBinaryName()} run <options>
|
|
23
22
|
$ ${getBinaryName()} deploy <options>
|
|
24
23
|
$ ${getBinaryName()} delete <options>
|
|
25
24
|
`);
|
|
@@ -29,35 +28,6 @@ function isDeleteCommand(command) {
|
|
|
29
28
|
return ["delete", "stop", "remove"].includes(command);
|
|
30
29
|
}
|
|
31
30
|
|
|
32
|
-
async function checkDeployEnv(env) {
|
|
33
|
-
const envStatus = await Promise.all(
|
|
34
|
-
Object.keys(env).map(async (key) => {
|
|
35
|
-
if (env[key] === SENSITIVE_VALUE) {
|
|
36
|
-
// check sensitive env
|
|
37
|
-
const res = await fetch(`${SKYNET_API_PREFIX}/secrets/verify?name=${key}`);
|
|
38
|
-
|
|
39
|
-
if (res.ok) {
|
|
40
|
-
const hasEnv = await res.json();
|
|
41
|
-
|
|
42
|
-
if (!hasEnv) {
|
|
43
|
-
return { name: key, ok: false };
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
return { name: key, ok: true };
|
|
49
|
-
}),
|
|
50
|
-
);
|
|
51
|
-
|
|
52
|
-
const missingEnvs = envStatus.filter((s) => !s.ok).map((s) => s.name);
|
|
53
|
-
|
|
54
|
-
if (missingEnvs.length > 0) {
|
|
55
|
-
console.log(`Missing the following sensitive environment on nomad server:${EOL}- ${missingEnvs.join(EOL + "- ")}`);
|
|
56
|
-
|
|
57
|
-
process.exit(1);
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
|
|
61
31
|
function checkAndSetEnv(env) {
|
|
62
32
|
const missingEnvs = [];
|
|
63
33
|
|
|
@@ -76,7 +46,7 @@ function checkAndSetEnv(env) {
|
|
|
76
46
|
}
|
|
77
47
|
}
|
|
78
48
|
|
|
79
|
-
function createApp({ parameterErrors, env,
|
|
49
|
+
function createApp({ parameterErrors, env, onRun, onDeploy }) {
|
|
80
50
|
if (parameterErrors.length > 0) {
|
|
81
51
|
console.log(`Parameter Validation Failed:${EOL}- ${parameterErrors.join(EOL + "- ")}`);
|
|
82
52
|
|
|
@@ -98,19 +68,9 @@ function createApp({ parameterErrors, env, check, onRun, onDeploy, onCheck }) {
|
|
|
98
68
|
process.argv.push("--stop");
|
|
99
69
|
}
|
|
100
70
|
|
|
101
|
-
|
|
102
|
-
await checkDeployEnv(env)
|
|
103
|
-
.then(() => onDeploy())
|
|
104
|
-
.catch(console.error);
|
|
105
|
-
} else {
|
|
106
|
-
await onDeploy();
|
|
107
|
-
}
|
|
108
|
-
} else if (!!check && subCommand === "check") {
|
|
109
|
-
checkAndSetEnv(env);
|
|
110
|
-
|
|
111
|
-
await onCheck();
|
|
71
|
+
await onDeploy();
|
|
112
72
|
} else {
|
|
113
|
-
printAppHelp(
|
|
73
|
+
printAppHelp();
|
|
114
74
|
}
|
|
115
75
|
};
|
|
116
76
|
}
|
|
@@ -147,24 +107,6 @@ function checkIndexerBuildParameter(build) {
|
|
|
147
107
|
return errors;
|
|
148
108
|
}
|
|
149
109
|
|
|
150
|
-
function checkCheckParameter(check) {
|
|
151
|
-
if (!check) {
|
|
152
|
-
return [];
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
const errors = [];
|
|
156
|
-
|
|
157
|
-
if (!check?.func) {
|
|
158
|
-
errors.push("must define check.func");
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
if (!check?.schedule) {
|
|
162
|
-
errors.push("must define check.schedule");
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
return errors;
|
|
166
|
-
}
|
|
167
|
-
|
|
168
110
|
function checkStateParameter(state) {
|
|
169
111
|
const errors = [];
|
|
170
112
|
|
|
@@ -179,11 +121,10 @@ function checkStateParameter(state) {
|
|
|
179
121
|
return errors;
|
|
180
122
|
}
|
|
181
123
|
|
|
182
|
-
function indexer({ name, selector, build,
|
|
124
|
+
function indexer({ name, selector, build, env = {}, region = "skynet-dc1" }) {
|
|
183
125
|
return createApp({
|
|
184
|
-
parameterErrors: [...checkIndexerBuildParameter(build), ...
|
|
126
|
+
parameterErrors: [...checkIndexerBuildParameter(build), ...checkEnvParameter(env)],
|
|
185
127
|
env,
|
|
186
|
-
check,
|
|
187
128
|
onRun: () => {
|
|
188
129
|
const { run } = createIndexerApp({
|
|
189
130
|
binaryName: `${getBinaryName()} run`,
|
|
@@ -213,29 +154,10 @@ function indexer({ name, selector, build, check, env = {}, region = "us-east-1"
|
|
|
213
154
|
killTimeout: build.killTimeout,
|
|
214
155
|
cpu: build.cpu,
|
|
215
156
|
mem: build.mem,
|
|
216
|
-
check: check && {
|
|
217
|
-
bin: `${bin} check`,
|
|
218
|
-
schedule: check.schedule,
|
|
219
|
-
killTimeout: check.killTimeout,
|
|
220
|
-
cpu: check.cpu || 100,
|
|
221
|
-
mem: check.mem || 100,
|
|
222
|
-
},
|
|
223
157
|
});
|
|
224
158
|
|
|
225
159
|
return deploy();
|
|
226
160
|
},
|
|
227
|
-
onCheck: () => {
|
|
228
|
-
const { monitor } = createMonitor({
|
|
229
|
-
binaryName: `${getBinaryName()} check`,
|
|
230
|
-
name,
|
|
231
|
-
type: "stateless",
|
|
232
|
-
selector,
|
|
233
|
-
check: check.func,
|
|
234
|
-
maxRetry: check.maxRetry,
|
|
235
|
-
});
|
|
236
|
-
|
|
237
|
-
return monitor();
|
|
238
|
-
},
|
|
239
161
|
});
|
|
240
162
|
}
|
|
241
163
|
|
|
@@ -277,17 +199,15 @@ function checkModeIndexerValidateParameter(validate) {
|
|
|
277
199
|
return errors;
|
|
278
200
|
}
|
|
279
201
|
|
|
280
|
-
function modeIndexer({ name, selector, state, build, validate,
|
|
202
|
+
function modeIndexer({ name, selector, state, build, validate, env = {}, region = "skynet-dc1" }) {
|
|
281
203
|
return createApp({
|
|
282
204
|
parameterErrors: [
|
|
283
205
|
...checkModeIndexerBuildParameter(build),
|
|
284
206
|
...checkModeIndexerValidateParameter(validate),
|
|
285
207
|
...checkStateParameter(state),
|
|
286
|
-
...checkCheckParameter(check),
|
|
287
208
|
...checkEnvParameter(env),
|
|
288
209
|
],
|
|
289
210
|
env,
|
|
290
|
-
check,
|
|
291
211
|
onRun: () => {
|
|
292
212
|
const { run } = createModeIndexerApp({
|
|
293
213
|
binaryName: `${getBinaryName()} run`,
|
|
@@ -332,37 +252,10 @@ function modeIndexer({ name, selector, state, build, validate, check, env = {},
|
|
|
332
252
|
validateKillTimeout: validate && validate.killTimeout,
|
|
333
253
|
validateCpu: validate && validate.cpu,
|
|
334
254
|
validateMem: validate && validate.mem,
|
|
335
|
-
|
|
336
|
-
check: check && {
|
|
337
|
-
bin: `${bin} check`,
|
|
338
|
-
schedule: check.schedule,
|
|
339
|
-
killTimeout: check.killTimeout,
|
|
340
|
-
cpu: check.cpu || 100,
|
|
341
|
-
mem: check.mem || 100,
|
|
342
|
-
},
|
|
343
255
|
});
|
|
344
256
|
|
|
345
257
|
return deploy();
|
|
346
258
|
},
|
|
347
|
-
onCheck: () => {
|
|
348
|
-
const { monitor } = createMonitor({
|
|
349
|
-
binaryName: `${getBinaryName()} check`,
|
|
350
|
-
name,
|
|
351
|
-
getState: async (nam, selectorFlags) => {
|
|
352
|
-
const latestId = await getIndexerLatestId(nam, selectorFlags);
|
|
353
|
-
const validatedId = await getIndexerValidatedId(nam, selectorFlags);
|
|
354
|
-
const buildState = await getIndexerState(nam, selectorFlags);
|
|
355
|
-
|
|
356
|
-
return { latestId, validatedId, buildState };
|
|
357
|
-
},
|
|
358
|
-
selector,
|
|
359
|
-
mode: true,
|
|
360
|
-
check: check.func,
|
|
361
|
-
maxRetry: check.maxRetry,
|
|
362
|
-
});
|
|
363
|
-
|
|
364
|
-
return monitor();
|
|
365
|
-
},
|
|
366
259
|
});
|
|
367
260
|
}
|
|
368
261
|
|
|
@@ -392,7 +285,7 @@ function checkProducerProduceParameter(produce) {
|
|
|
392
285
|
return errors;
|
|
393
286
|
}
|
|
394
287
|
|
|
395
|
-
function producer({ name, selector, produce,
|
|
288
|
+
function producer({ name, selector, produce, state, env = {}, region = "skynet-dc1" }) {
|
|
396
289
|
const envWithDefaultValues = {
|
|
397
290
|
SKYNET_KAFKA_SERVER: SENSITIVE_VALUE,
|
|
398
291
|
SKYNET_KAFKA_USERNAME: SENSITIVE_VALUE,
|
|
@@ -404,11 +297,9 @@ function producer({ name, selector, produce, check, state, env = {}, region = "u
|
|
|
404
297
|
parameterErrors: [
|
|
405
298
|
...checkProducerProduceParameter(produce),
|
|
406
299
|
...checkStateParameter(state),
|
|
407
|
-
...checkCheckParameter(check),
|
|
408
300
|
...checkEnvParameter(env),
|
|
409
301
|
],
|
|
410
302
|
env: envWithDefaultValues,
|
|
411
|
-
check,
|
|
412
303
|
onRun: () => {
|
|
413
304
|
const { run } = createProducerApp({
|
|
414
305
|
binaryName: `${getBinaryName()} run`,
|
|
@@ -445,33 +336,10 @@ function producer({ name, selector, produce, check, state, env = {}, region = "u
|
|
|
445
336
|
killTimeout: produce.killTimeout,
|
|
446
337
|
cpu: produce.cpu,
|
|
447
338
|
mem: produce.mem,
|
|
448
|
-
check: check && {
|
|
449
|
-
bin: `${bin} check`,
|
|
450
|
-
schedule: check.schedule,
|
|
451
|
-
killTimeout: check.killTimeout,
|
|
452
|
-
cpu: check.cpu || 100,
|
|
453
|
-
mem: check.mem || 100,
|
|
454
|
-
},
|
|
455
339
|
});
|
|
456
340
|
|
|
457
341
|
return deploy();
|
|
458
342
|
},
|
|
459
|
-
onCheck: () => {
|
|
460
|
-
const { monitor } = createMonitor({
|
|
461
|
-
binaryName: `${getBinaryName()} check`,
|
|
462
|
-
name,
|
|
463
|
-
getState: async (nam, selectorFlags) => {
|
|
464
|
-
const latestId = await getProducerLatestId(nam, selectorFlags);
|
|
465
|
-
|
|
466
|
-
return { latestId };
|
|
467
|
-
},
|
|
468
|
-
selector,
|
|
469
|
-
check: check.func,
|
|
470
|
-
maxRetry: check.maxRetry,
|
|
471
|
-
});
|
|
472
|
-
|
|
473
|
-
return monitor();
|
|
474
|
-
},
|
|
475
343
|
});
|
|
476
344
|
}
|
|
477
345
|
|
|
@@ -497,7 +365,7 @@ function checkConsumerConsumeParameter(consume) {
|
|
|
497
365
|
return errors;
|
|
498
366
|
}
|
|
499
367
|
|
|
500
|
-
function consumer({ name, selector, consume,
|
|
368
|
+
function consumer({ name, selector, consume, env = {}, region = "skynet-dc1" }) {
|
|
501
369
|
const envWithDefaultValues = {
|
|
502
370
|
SKYNET_KAFKA_SERVER: SENSITIVE_VALUE,
|
|
503
371
|
SKYNET_KAFKA_USERNAME: SENSITIVE_VALUE,
|
|
@@ -506,13 +374,8 @@ function consumer({ name, selector, consume, check, env = {}, region = "us-east-
|
|
|
506
374
|
};
|
|
507
375
|
|
|
508
376
|
return createApp({
|
|
509
|
-
parameterErrors: [
|
|
510
|
-
...checkConsumerConsumeParameter(consume),
|
|
511
|
-
...checkCheckParameter(check),
|
|
512
|
-
...checkEnvParameter(env),
|
|
513
|
-
],
|
|
377
|
+
parameterErrors: [...checkConsumerConsumeParameter(consume), ...checkEnvParameter(env)],
|
|
514
378
|
env: envWithDefaultValues,
|
|
515
|
-
check,
|
|
516
379
|
onRun: () => {
|
|
517
380
|
const { run } = createConsumerApp({
|
|
518
381
|
binaryName: `${getBinaryName()} run`,
|
|
@@ -545,28 +408,10 @@ function consumer({ name, selector, consume, check, env = {}, region = "us-east-
|
|
|
545
408
|
killTimeout: consume.killTimeout,
|
|
546
409
|
cpu: consume.cpu,
|
|
547
410
|
mem: consume.mem,
|
|
548
|
-
check: check && {
|
|
549
|
-
bin: `${bin} check`,
|
|
550
|
-
schedule: check.schedule,
|
|
551
|
-
killTimeout: check.killTimeout,
|
|
552
|
-
cpu: check.cpu || 100,
|
|
553
|
-
mem: check.mem || 100,
|
|
554
|
-
},
|
|
555
411
|
});
|
|
556
412
|
|
|
557
413
|
return deploy();
|
|
558
414
|
},
|
|
559
|
-
onCheck: () => {
|
|
560
|
-
const { monitor } = createMonitor({
|
|
561
|
-
binaryName: `${getBinaryName()} check`,
|
|
562
|
-
name,
|
|
563
|
-
selector,
|
|
564
|
-
check: check.func,
|
|
565
|
-
maxRetry: check.maxRetry,
|
|
566
|
-
});
|
|
567
|
-
|
|
568
|
-
return monitor();
|
|
569
|
-
},
|
|
570
415
|
});
|
|
571
416
|
}
|
|
572
417
|
|
|
@@ -624,49 +469,17 @@ function checkApiRoutesParameter(routes) {
|
|
|
624
469
|
return errors;
|
|
625
470
|
}
|
|
626
471
|
|
|
627
|
-
function api({ name, routes, serve, beforeListen, env = {}, region = "
|
|
472
|
+
function api({ name, routes, serve, beforeListen, env = {}, region = "skynet-dc1" }) {
|
|
628
473
|
// do not support selector for now
|
|
629
474
|
const selector = {};
|
|
630
475
|
|
|
631
|
-
// hard code a simple health check for now
|
|
632
|
-
const check = {
|
|
633
|
-
func: async () => {
|
|
634
|
-
const errors = [];
|
|
635
|
-
const url = `http://localhost:9999${serve.prefix}/`;
|
|
636
|
-
|
|
637
|
-
try {
|
|
638
|
-
const res = await fetch(url);
|
|
639
|
-
|
|
640
|
-
if (!res.ok) {
|
|
641
|
-
errors.push({
|
|
642
|
-
type: ERROR_LEVEL.ERROR,
|
|
643
|
-
message: `service ${name} is unhealthy`,
|
|
644
|
-
});
|
|
645
|
-
}
|
|
646
|
-
} catch (fetchErr) {
|
|
647
|
-
console.log(`error fetching ${url}`, fetchErr);
|
|
648
|
-
|
|
649
|
-
errors.push({
|
|
650
|
-
type: ERROR_LEVEL.ERROR,
|
|
651
|
-
message: `service ${name} is unhealthy`,
|
|
652
|
-
});
|
|
653
|
-
}
|
|
654
|
-
|
|
655
|
-
return errors;
|
|
656
|
-
},
|
|
657
|
-
schedule: every(5).minutes,
|
|
658
|
-
maxRetry: 0,
|
|
659
|
-
};
|
|
660
|
-
|
|
661
476
|
return createApp({
|
|
662
477
|
parameterErrors: [
|
|
663
478
|
...checkApiRoutesParameter(routes),
|
|
664
479
|
...checkApiServeParameter(serve, routes),
|
|
665
|
-
...checkCheckParameter(check),
|
|
666
480
|
...checkEnvParameter(env),
|
|
667
481
|
],
|
|
668
482
|
env,
|
|
669
|
-
check,
|
|
670
483
|
onRun: () => {
|
|
671
484
|
process.title = name;
|
|
672
485
|
|
|
@@ -705,28 +518,10 @@ function api({ name, routes, serve, beforeListen, env = {}, region = "us-east-1"
|
|
|
705
518
|
prefix: serve.prefix,
|
|
706
519
|
port: serve.port,
|
|
707
520
|
},
|
|
708
|
-
check: check && {
|
|
709
|
-
bin: `${bin} check`,
|
|
710
|
-
schedule: check.schedule,
|
|
711
|
-
killTimeout: check.killTimeout,
|
|
712
|
-
cpu: check.cpu || 100,
|
|
713
|
-
mem: check.mem || 100,
|
|
714
|
-
},
|
|
715
521
|
});
|
|
716
522
|
|
|
717
523
|
return deploy();
|
|
718
524
|
},
|
|
719
|
-
onCheck: () => {
|
|
720
|
-
const { monitor } = createMonitor({
|
|
721
|
-
binaryName: `${getBinaryName()} check`,
|
|
722
|
-
name,
|
|
723
|
-
selector,
|
|
724
|
-
check: check.func,
|
|
725
|
-
maxRetry: check.maxRetry,
|
|
726
|
-
});
|
|
727
|
-
|
|
728
|
-
return monitor();
|
|
729
|
-
},
|
|
730
525
|
});
|
|
731
526
|
}
|
|
732
527
|
|
package/const.d.ts
CHANGED
package/const.js
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
import { getNodeRealApiKey, ensureAndGet } from "./env.js";
|
|
2
2
|
|
|
3
|
-
const SKYNET_API_PREFIX = "https://api.certik-skynet.com";
|
|
4
|
-
|
|
5
3
|
const PROTOCOLS = {
|
|
6
4
|
eth: {
|
|
7
5
|
nativeTokenName: "Ethereum",
|
|
@@ -138,4 +136,4 @@ const TIME = {
|
|
|
138
136
|
},
|
|
139
137
|
};
|
|
140
138
|
|
|
141
|
-
export {
|
|
139
|
+
export { PROTOCOLS, TIME };
|
package/deploy.js
CHANGED
|
@@ -21,28 +21,6 @@ const INTERVAL_ALIASES = {
|
|
|
21
21
|
"@weekly": "0 0 0 * * 0 *",
|
|
22
22
|
};
|
|
23
23
|
|
|
24
|
-
function buildEnvTemplate(additionalEnv, isProduction) {
|
|
25
|
-
return Object.keys(additionalEnv)
|
|
26
|
-
.map((key) => {
|
|
27
|
-
return `${key}=${
|
|
28
|
-
additionalEnv[key] ? `"${additionalEnv[key]}"` : getEnvironmentVariableValue(key, isProduction)
|
|
29
|
-
}`;
|
|
30
|
-
})
|
|
31
|
-
.join("\n");
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
function getEnvironmentVariableValue(name, isProduction) {
|
|
35
|
-
if (isProduction) {
|
|
36
|
-
return `{{key "secrets/${name}" | toJSON}}`;
|
|
37
|
-
} else {
|
|
38
|
-
if (!process.env[name]) {
|
|
39
|
-
return `""`;
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
return `"${process.env[name]}"`.replaceAll("\n", "\\n");
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
|
|
46
24
|
const genConfig = ({
|
|
47
25
|
jobName,
|
|
48
26
|
workingDirectory,
|
|
@@ -56,7 +34,7 @@ const genConfig = ({
|
|
|
56
34
|
service,
|
|
57
35
|
additionalEnv = [],
|
|
58
36
|
type = "batch",
|
|
59
|
-
region = "
|
|
37
|
+
region = "skynet-dc1",
|
|
60
38
|
isProduction,
|
|
61
39
|
}) => `job "${jobName}" {
|
|
62
40
|
datacenters = ["${region}"]
|
|
@@ -85,9 +63,7 @@ const genConfig = ({
|
|
|
85
63
|
|
|
86
64
|
group "default" {
|
|
87
65
|
${count && count > 1 ? `count = ${count}` : ""}
|
|
88
|
-
${
|
|
89
|
-
count && count > 1
|
|
90
|
-
? `# Rolling Update
|
|
66
|
+
${count && count > 1 ? `# Rolling Update
|
|
91
67
|
update {
|
|
92
68
|
max_parallel = 1
|
|
93
69
|
min_healthy_time = "10s"
|
|
@@ -118,7 +94,7 @@ const genConfig = ({
|
|
|
118
94
|
command = "sh"
|
|
119
95
|
args = [
|
|
120
96
|
"-c",
|
|
121
|
-
"cd \${meta.skynet_code_path}/${workingDirectory} && if [ -e bun.lockb ]; then bun install --silent; else yarn install --silent; fi&& exec ${cmd}"
|
|
97
|
+
"cd \${meta.skynet_code_path}/${workingDirectory} && if [ -e bun.lockb ]; then bun install --silent; else yarn install --silent; fi && exec ${cmd}"
|
|
122
98
|
]
|
|
123
99
|
}
|
|
124
100
|
|
|
@@ -144,22 +120,26 @@ const genConfig = ({
|
|
|
144
120
|
: ""
|
|
145
121
|
}
|
|
146
122
|
|
|
123
|
+
# doppler integration support
|
|
124
|
+
# it is always there but a project can decide to not use it
|
|
147
125
|
template {
|
|
148
126
|
change_mode = "restart"
|
|
149
127
|
destination = "secrets/context.env"
|
|
150
128
|
env = true
|
|
151
|
-
|
|
152
|
-
data = <<EOH
|
|
153
|
-
${buildEnvTemplate(additionalEnv, isProduction)}
|
|
154
|
-
EOH
|
|
129
|
+
data = "DOPPLER_TOKEN={{key \\"infra-nomad/doppler-token\\"}}"
|
|
155
130
|
}
|
|
156
131
|
|
|
157
|
-
#
|
|
158
|
-
# available to the task when it runs.
|
|
159
|
-
# always update environment so that new deployment always triggers
|
|
132
|
+
# always update SKYNET_DEPLOYED_AT so that new deployment always triggers
|
|
160
133
|
env {
|
|
161
|
-
SKYNET_ENVIRONMENT="${isProduction ? "prd" : "dev"}"
|
|
162
134
|
SKYNET_DEPLOYED_AT="${new Date().toISOString()}"
|
|
135
|
+
HOME="/root"
|
|
136
|
+
DOPPLER_PROJECT="${workingDirectory}"
|
|
137
|
+
DOPPLER_CONFIG="${isProduction ? "prd" : "dev"}"
|
|
138
|
+
SKYNET_ENVIRONMENT="${isProduction ? "prd" : "dev"}"
|
|
139
|
+
${Object.entries(additionalEnv)
|
|
140
|
+
.filter(([k, v]) => !!v)
|
|
141
|
+
.map(([key, value]) => `${key}="${value}"`)
|
|
142
|
+
.join(" \n")}
|
|
163
143
|
}
|
|
164
144
|
|
|
165
145
|
kill_timeout = "${killTimeout || "60s"}"
|
|
@@ -219,7 +199,7 @@ async function getNomadAddr(isProduction) {
|
|
|
219
199
|
let nomadAddr;
|
|
220
200
|
|
|
221
201
|
if (isProduction) {
|
|
222
|
-
nomadAddr = getEnvOrThrow("
|
|
202
|
+
nomadAddr = getEnvOrThrow("NOMAD_ADDR");
|
|
223
203
|
} else {
|
|
224
204
|
nomadAddr = "http://127.0.0.1:4646";
|
|
225
205
|
}
|
|
@@ -294,8 +274,7 @@ function createModeDeploy({
|
|
|
294
274
|
bin = "bin/indexer",
|
|
295
275
|
selector = {},
|
|
296
276
|
env = {},
|
|
297
|
-
region = "
|
|
298
|
-
check,
|
|
277
|
+
region = "skynet-dc1",
|
|
299
278
|
deltaSchedule,
|
|
300
279
|
validateSchedule,
|
|
301
280
|
deltaKillTimeout,
|
|
@@ -398,32 +377,6 @@ function createModeDeploy({
|
|
|
398
377
|
const nomadAddr = await getNomadAddr(production);
|
|
399
378
|
|
|
400
379
|
await runNomadJob(nomadPath, nomadAddr, jobName, mainJobDefinition, stop, dryRun);
|
|
401
|
-
|
|
402
|
-
if (check && check.bin) {
|
|
403
|
-
console.log("");
|
|
404
|
-
|
|
405
|
-
const monitorJobName = `${jobName}-monitor`;
|
|
406
|
-
const monitorJobDefinition = genConfig({
|
|
407
|
-
jobName: monitorJobName,
|
|
408
|
-
cron: INTERVAL_ALIASES[check.schedule] || check.schedule,
|
|
409
|
-
workingDirectory,
|
|
410
|
-
additionalEnv: {
|
|
411
|
-
...env,
|
|
412
|
-
SKYNET_NOMAD_PRODUCTION_ADDR: null,
|
|
413
|
-
SKYNET_SLACK_TOKEN: null,
|
|
414
|
-
OPSGENIE_API_KEY: null,
|
|
415
|
-
OPSGENIE_END_POINT: null,
|
|
416
|
-
},
|
|
417
|
-
region,
|
|
418
|
-
cmd: `${check.bin} ${args} ${production ? "--production" : ""}`,
|
|
419
|
-
killTimeout: check.killTimeout || "10s",
|
|
420
|
-
cpu: check.cpu || 500,
|
|
421
|
-
mem: check.mem || 500,
|
|
422
|
-
isProduction: production,
|
|
423
|
-
});
|
|
424
|
-
|
|
425
|
-
await runNomadJob(nomadPath, nomadAddr, monitorJobName, monitorJobDefinition, stop, dryRun);
|
|
426
|
-
}
|
|
427
380
|
}
|
|
428
381
|
|
|
429
382
|
function deploy() {
|
|
@@ -512,11 +465,10 @@ function createDeploy({
|
|
|
512
465
|
workingDirectory,
|
|
513
466
|
bin = "bin/indexer",
|
|
514
467
|
selector = {},
|
|
515
|
-
region = "
|
|
468
|
+
region = "skynet-dc1",
|
|
516
469
|
type = "batch",
|
|
517
470
|
env = {},
|
|
518
471
|
count,
|
|
519
|
-
check,
|
|
520
472
|
schedule,
|
|
521
473
|
restart,
|
|
522
474
|
killTimeout,
|
|
@@ -566,32 +518,6 @@ function createDeploy({
|
|
|
566
518
|
const nomadAddr = await getNomadAddr(production);
|
|
567
519
|
|
|
568
520
|
await runNomadJob(nomadPath, nomadAddr, jobName, nomadJobDefinition, stop, dryRun);
|
|
569
|
-
|
|
570
|
-
if (check && check.bin) {
|
|
571
|
-
console.log("");
|
|
572
|
-
|
|
573
|
-
const monitorJobName = `${jobName}-monitor`;
|
|
574
|
-
const monitorJobDefinition = genConfig({
|
|
575
|
-
jobName: monitorJobName,
|
|
576
|
-
cron: INTERVAL_ALIASES[check.schedule] || check.schedule,
|
|
577
|
-
workingDirectory,
|
|
578
|
-
additionalEnv: {
|
|
579
|
-
...env,
|
|
580
|
-
SKYNET_NOMAD_PRODUCTION_ADDR: null,
|
|
581
|
-
SKYNET_SLACK_TOKEN: null,
|
|
582
|
-
OPSGENIE_API_KEY: null,
|
|
583
|
-
OPSGENIE_END_POINT: null,
|
|
584
|
-
},
|
|
585
|
-
region,
|
|
586
|
-
cmd: `${check.bin} ${args} ${production ? "--production" : ""}`,
|
|
587
|
-
killTimeout: check.killTimeout || "10s",
|
|
588
|
-
cpu: check.cpu || 500,
|
|
589
|
-
mem: check.mem || 500,
|
|
590
|
-
isProduction: production,
|
|
591
|
-
});
|
|
592
|
-
|
|
593
|
-
await runNomadJob(nomadPath, nomadAddr, monitorJobName, monitorJobDefinition, stop, dryRun);
|
|
594
|
-
}
|
|
595
521
|
}
|
|
596
522
|
|
|
597
523
|
function deploy() {
|
package/examples/api.js
CHANGED
package/examples/consumer.js
CHANGED
|
@@ -4,26 +4,15 @@
|
|
|
4
4
|
// a few test commands to try on
|
|
5
5
|
// $ examples/consumer run --protocol bsc --verbose
|
|
6
6
|
// $ examples/consumer run --protocol eth
|
|
7
|
-
// $ examples/consumer check --protocol eth
|
|
8
7
|
// $ examples/consumer deploy --protocol eth
|
|
9
8
|
// $ examples/consumer --help
|
|
10
9
|
|
|
11
|
-
import { consumer, every, ERROR_LEVEL } from "../app";
|
|
10
|
+
import { consumer, every, ERROR_LEVEL } from "../app.js";
|
|
12
11
|
|
|
13
12
|
async function consume({ protocol, messages, verbose }) {
|
|
14
13
|
console.log("consume called with", protocol, messages, verbose);
|
|
15
14
|
}
|
|
16
15
|
|
|
17
|
-
async function check({ protocol, state, verbose }) {
|
|
18
|
-
console.log("check called with", protocol, state, verbose);
|
|
19
|
-
|
|
20
|
-
const errors = [];
|
|
21
|
-
|
|
22
|
-
errors.push({ type: ERROR_LEVEL.CRITICAL, message: "processed height lagged behind" });
|
|
23
|
-
|
|
24
|
-
return errors;
|
|
25
|
-
}
|
|
26
|
-
|
|
27
16
|
const app = consumer({
|
|
28
17
|
name: "example-consumer",
|
|
29
18
|
selector: { protocol: { type: "string", description: "from which chain to consume data" } },
|
|
@@ -34,13 +23,7 @@ const app = consumer({
|
|
|
34
23
|
maxRetry: 1,
|
|
35
24
|
|
|
36
25
|
cpu: 600,
|
|
37
|
-
mem:
|
|
38
|
-
},
|
|
39
|
-
|
|
40
|
-
check: {
|
|
41
|
-
func: check,
|
|
42
|
-
schedule: every(1).minute,
|
|
43
|
-
slackChannel: "skynet-notifications-local-dev",
|
|
26
|
+
mem: 600,
|
|
44
27
|
},
|
|
45
28
|
});
|
|
46
29
|
|
package/examples/indexer.js
CHANGED
|
@@ -4,11 +4,10 @@
|
|
|
4
4
|
// a few test commands to try on
|
|
5
5
|
// $ examples/indexer run --protocol bsc --verbose
|
|
6
6
|
// $ examples/indexer run --protocol eth
|
|
7
|
-
// $ examples/indexer check --protocol eth
|
|
8
7
|
// $ examples/indexer deploy --protocol eth
|
|
9
8
|
// $ examples/indexer --help
|
|
10
9
|
|
|
11
|
-
import { indexer, every, ERROR_LEVEL } from "../app";
|
|
10
|
+
import { indexer, every, ERROR_LEVEL } from "../app.js";
|
|
12
11
|
|
|
13
12
|
async function build({ protocol, verbose }) {
|
|
14
13
|
console.log("build called with", protocol, verbose);
|
|
@@ -18,22 +17,6 @@ async function build({ protocol, verbose }) {
|
|
|
18
17
|
}
|
|
19
18
|
}
|
|
20
19
|
|
|
21
|
-
async function check({ protocol, state, verbose }) {
|
|
22
|
-
console.log("check called with", protocol, state, verbose);
|
|
23
|
-
|
|
24
|
-
// latestId
|
|
25
|
-
// latestIdOnChain
|
|
26
|
-
|
|
27
|
-
const errors = [];
|
|
28
|
-
|
|
29
|
-
errors.push({
|
|
30
|
-
type: ERROR_LEVEL.WARNING,
|
|
31
|
-
message: "processed *height* lagged behind", // can be markdown
|
|
32
|
-
});
|
|
33
|
-
|
|
34
|
-
return errors;
|
|
35
|
-
}
|
|
36
|
-
|
|
37
20
|
const app = indexer({
|
|
38
21
|
name: "example-indexer",
|
|
39
22
|
|
|
@@ -45,21 +28,16 @@ const app = indexer({
|
|
|
45
28
|
},
|
|
46
29
|
},
|
|
47
30
|
|
|
48
|
-
env: {
|
|
31
|
+
env: {
|
|
32
|
+
MY_FIXED_VAR: "fixed",
|
|
33
|
+
},
|
|
49
34
|
|
|
50
35
|
build: {
|
|
51
36
|
func: build,
|
|
52
37
|
schedule: every(1).minute,
|
|
53
38
|
killTimeout: "90s",
|
|
54
39
|
cpu: 600,
|
|
55
|
-
mem:
|
|
56
|
-
},
|
|
57
|
-
|
|
58
|
-
check: {
|
|
59
|
-
func: check,
|
|
60
|
-
schedule: every(2).minutes,
|
|
61
|
-
maxRetry: 0,
|
|
62
|
-
slackChannel: "skynet-notifications-local-dev",
|
|
40
|
+
mem: 600,
|
|
63
41
|
},
|
|
64
42
|
});
|
|
65
43
|
|
package/examples/mode-indexer.js
CHANGED
|
@@ -5,11 +5,10 @@
|
|
|
5
5
|
// $ examples/mode-indexer run --protocol bsc --verbose
|
|
6
6
|
// $ examples/mode-indexer run --mode rebuild --protocol bsc --verbose
|
|
7
7
|
// $ examples/mode-indexer run --protocol eth
|
|
8
|
-
// $ examples/mode-indexer check --protocol eth
|
|
9
8
|
// $ examples/mode-indexer deploy --protocol eth
|
|
10
9
|
// $ examples/mode-indexer --help
|
|
11
10
|
|
|
12
|
-
import { modeIndexer, every, SENSITIVE_VALUE } from "../app";
|
|
11
|
+
import { modeIndexer, every, SENSITIVE_VALUE } from "../app.js";
|
|
13
12
|
|
|
14
13
|
async function build({ protocol, from, to, verbose }) {
|
|
15
14
|
console.log("build called with", protocol, from, to, verbose);
|
|
@@ -19,18 +18,6 @@ async function validate({ protocol, from, to, verbose }) {
|
|
|
19
18
|
console.log("validate called with", protocol, from, to, verbose);
|
|
20
19
|
}
|
|
21
20
|
|
|
22
|
-
async function check({ mode, protocol, state, verbose }) {
|
|
23
|
-
console.log("check called with", mode, protocol, state, verbose);
|
|
24
|
-
|
|
25
|
-
const errors = [];
|
|
26
|
-
|
|
27
|
-
// if (state.latestId < getBlockLatestHeight(protocol)) {
|
|
28
|
-
// errors.push({ type: ERROR_LEVEL.INFO, message: "processed height lagged behind" });
|
|
29
|
-
// }
|
|
30
|
-
|
|
31
|
-
return errors;
|
|
32
|
-
}
|
|
33
|
-
|
|
34
21
|
const app = modeIndexer({
|
|
35
22
|
name: "example-mode-indexer",
|
|
36
23
|
|
|
@@ -59,7 +46,7 @@ const app = modeIndexer({
|
|
|
59
46
|
|
|
60
47
|
schedule: every(1).minute,
|
|
61
48
|
cpu: 800,
|
|
62
|
-
mem:
|
|
49
|
+
mem: 800,
|
|
63
50
|
},
|
|
64
51
|
|
|
65
52
|
validate: {
|
|
@@ -71,12 +58,6 @@ const app = modeIndexer({
|
|
|
71
58
|
cpu: 600,
|
|
72
59
|
mem: 200,
|
|
73
60
|
},
|
|
74
|
-
|
|
75
|
-
check: {
|
|
76
|
-
func: check,
|
|
77
|
-
schedule: every(2).minutes,
|
|
78
|
-
slackChannel: "skynet-notifications-local-dev",
|
|
79
|
-
},
|
|
80
61
|
});
|
|
81
62
|
|
|
82
63
|
app();
|
package/examples/producer.js
CHANGED
|
@@ -4,11 +4,10 @@
|
|
|
4
4
|
// a few test commands to try on
|
|
5
5
|
// $ examples/producer run --protocol bsc --verbose
|
|
6
6
|
// $ examples/producer run --protocol eth
|
|
7
|
-
// $ examples/producer check --protocol eth
|
|
8
7
|
// $ examples/producer deploy --protocol eth
|
|
9
8
|
// $ examples/producer --help
|
|
10
9
|
|
|
11
|
-
import { producer, every, SENSITIVE_VALUE, ERROR_LEVEL } from "../app";
|
|
10
|
+
import { producer, every, SENSITIVE_VALUE, ERROR_LEVEL } from "../app.js";
|
|
12
11
|
|
|
13
12
|
async function produce({ protocol, from, to, verbose, send }) {
|
|
14
13
|
console.log("produce called with", protocol, from, to, verbose);
|
|
@@ -21,16 +20,6 @@ async function produce({ protocol, from, to, verbose, send }) {
|
|
|
21
20
|
await send(items);
|
|
22
21
|
}
|
|
23
22
|
|
|
24
|
-
async function check({ protocol, state, verbose }) {
|
|
25
|
-
console.log("check called with", protocol, state, verbose);
|
|
26
|
-
|
|
27
|
-
const errors = [];
|
|
28
|
-
|
|
29
|
-
errors.push({ type: ERROR_LEVEL.CRITICAL, message: "producer error" });
|
|
30
|
-
|
|
31
|
-
return errors;
|
|
32
|
-
}
|
|
33
|
-
|
|
34
23
|
const app = producer({
|
|
35
24
|
name: "example-producer",
|
|
36
25
|
selector: { protocol: { type: "string", description: "for which chain to produce data" } },
|
|
@@ -67,13 +56,7 @@ const app = producer({
|
|
|
67
56
|
maxRetry: 1, // default 2
|
|
68
57
|
|
|
69
58
|
cpu: 600,
|
|
70
|
-
mem:
|
|
71
|
-
},
|
|
72
|
-
|
|
73
|
-
check: {
|
|
74
|
-
func: check,
|
|
75
|
-
schedule: every(2).minutes,
|
|
76
|
-
slackChannel: "skynet-notifications-local-dev",
|
|
59
|
+
mem: 600,
|
|
77
60
|
},
|
|
78
61
|
});
|
|
79
62
|
|