@certik/skynet 0.25.1 → 0.25.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.vscode/settings.json +5 -0
- package/CHANGELOG.md +6 -0
- package/dist/api.js +38 -38
- package/dist/app.js +285 -285
- package/dist/deploy.js +20 -20
- package/dist/dynamodb.js +22 -22
- package/dist/indexer.js +132 -132
- package/examples/api.ts +0 -0
- package/examples/indexer.ts +0 -0
- package/examples/mode-indexer.ts +0 -0
- package/package.json +1 -1
- package/src/deploy.ts +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.25.2
|
|
4
|
+
|
|
5
|
+
- Update deploy command to check for bun.lock file.
|
|
6
|
+
- Use --production flag when installing packages.
|
|
7
|
+
|
|
3
8
|
## 0.25.1
|
|
4
9
|
|
|
5
10
|
- Updated README.md for deployment instructions
|
|
@@ -64,6 +69,7 @@
|
|
|
64
69
|
## 0.20.0
|
|
65
70
|
|
|
66
71
|
- Fix databricks with bun (extra step needed for projects using the lib):
|
|
72
|
+
|
|
67
73
|
- Copy the patch file `patches/@databricks%2Fsql@1.9.0.patch`
|
|
68
74
|
- Add the following configs to your `package.json`:
|
|
69
75
|
|
package/dist/api.js
CHANGED
|
@@ -1,3 +1,41 @@
|
|
|
1
|
+
// src/selector.ts
|
|
2
|
+
function getSelectorDesc(selector) {
|
|
3
|
+
return Object.keys(selector).map((name) => {
|
|
4
|
+
return ` --${name.padEnd(14)}${selector[name].desc || selector[name].description || ""}`;
|
|
5
|
+
}).join(`
|
|
6
|
+
`);
|
|
7
|
+
}
|
|
8
|
+
function getSelectorFlags(selector) {
|
|
9
|
+
return Object.keys(selector).reduce((acc, name) => {
|
|
10
|
+
const flag = {
|
|
11
|
+
type: selector[name].type || "string",
|
|
12
|
+
...selector[name]
|
|
13
|
+
};
|
|
14
|
+
if (!selector[name].optional && selector[name].isRequired !== false) {
|
|
15
|
+
flag.isRequired = true;
|
|
16
|
+
}
|
|
17
|
+
return { ...acc, [name]: flag };
|
|
18
|
+
}, {});
|
|
19
|
+
}
|
|
20
|
+
function toSelectorString(selectorFlags, delim = ",") {
|
|
21
|
+
return Object.keys(selectorFlags).sort().map((flag) => {
|
|
22
|
+
return `${flag}=${selectorFlags[flag]}`;
|
|
23
|
+
}).join(delim);
|
|
24
|
+
}
|
|
25
|
+
function normalizeSelectorValue(v) {
|
|
26
|
+
return v.replace(/[^A-Za-z0-9]+/g, "-");
|
|
27
|
+
}
|
|
28
|
+
function getJobName(name, selectorFlags, mode) {
|
|
29
|
+
const selectorNamePart = Object.keys(selectorFlags).sort().map((name2) => selectorFlags[name2]).join("-");
|
|
30
|
+
let jobName = name;
|
|
31
|
+
if (mode) {
|
|
32
|
+
jobName += `-${mode}`;
|
|
33
|
+
}
|
|
34
|
+
if (selectorNamePart.length > 0) {
|
|
35
|
+
jobName += `-${normalizeSelectorValue(selectorNamePart)}`;
|
|
36
|
+
}
|
|
37
|
+
return jobName;
|
|
38
|
+
}
|
|
1
39
|
// src/env.ts
|
|
2
40
|
function ensureAndGet(envName, defaultValue) {
|
|
3
41
|
return process.env[envName] || defaultValue;
|
|
@@ -74,44 +112,6 @@ var logger = {
|
|
|
74
112
|
}
|
|
75
113
|
}
|
|
76
114
|
};
|
|
77
|
-
// src/selector.ts
|
|
78
|
-
function getSelectorDesc(selector) {
|
|
79
|
-
return Object.keys(selector).map((name) => {
|
|
80
|
-
return ` --${name.padEnd(14)}${selector[name].desc || selector[name].description || ""}`;
|
|
81
|
-
}).join(`
|
|
82
|
-
`);
|
|
83
|
-
}
|
|
84
|
-
function getSelectorFlags(selector) {
|
|
85
|
-
return Object.keys(selector).reduce((acc, name) => {
|
|
86
|
-
const flag = {
|
|
87
|
-
type: selector[name].type || "string",
|
|
88
|
-
...selector[name]
|
|
89
|
-
};
|
|
90
|
-
if (!selector[name].optional && selector[name].isRequired !== false) {
|
|
91
|
-
flag.isRequired = true;
|
|
92
|
-
}
|
|
93
|
-
return { ...acc, [name]: flag };
|
|
94
|
-
}, {});
|
|
95
|
-
}
|
|
96
|
-
function toSelectorString(selectorFlags, delim = ",") {
|
|
97
|
-
return Object.keys(selectorFlags).sort().map((flag) => {
|
|
98
|
-
return `${flag}=${selectorFlags[flag]}`;
|
|
99
|
-
}).join(delim);
|
|
100
|
-
}
|
|
101
|
-
function normalizeSelectorValue(v) {
|
|
102
|
-
return v.replace(/[^A-Za-z0-9]+/g, "-");
|
|
103
|
-
}
|
|
104
|
-
function getJobName(name, selectorFlags, mode) {
|
|
105
|
-
const selectorNamePart = Object.keys(selectorFlags).sort().map((name2) => selectorFlags[name2]).join("-");
|
|
106
|
-
let jobName = name;
|
|
107
|
-
if (mode) {
|
|
108
|
-
jobName += `-${mode}`;
|
|
109
|
-
}
|
|
110
|
-
if (selectorNamePart.length > 0) {
|
|
111
|
-
jobName += `-${normalizeSelectorValue(selectorNamePart)}`;
|
|
112
|
-
}
|
|
113
|
-
return jobName;
|
|
114
|
-
}
|
|
115
115
|
// src/api.ts
|
|
116
116
|
import osModule from "os";
|
|
117
117
|
import express from "express";
|
package/dist/app.js
CHANGED
|
@@ -1,3 +1,41 @@
|
|
|
1
|
+
// src/selector.ts
|
|
2
|
+
function getSelectorDesc(selector) {
|
|
3
|
+
return Object.keys(selector).map((name) => {
|
|
4
|
+
return ` --${name.padEnd(14)}${selector[name].desc || selector[name].description || ""}`;
|
|
5
|
+
}).join(`
|
|
6
|
+
`);
|
|
7
|
+
}
|
|
8
|
+
function getSelectorFlags(selector) {
|
|
9
|
+
return Object.keys(selector).reduce((acc, name) => {
|
|
10
|
+
const flag = {
|
|
11
|
+
type: selector[name].type || "string",
|
|
12
|
+
...selector[name]
|
|
13
|
+
};
|
|
14
|
+
if (!selector[name].optional && selector[name].isRequired !== false) {
|
|
15
|
+
flag.isRequired = true;
|
|
16
|
+
}
|
|
17
|
+
return { ...acc, [name]: flag };
|
|
18
|
+
}, {});
|
|
19
|
+
}
|
|
20
|
+
function toSelectorString(selectorFlags, delim = ",") {
|
|
21
|
+
return Object.keys(selectorFlags).sort().map((flag) => {
|
|
22
|
+
return `${flag}=${selectorFlags[flag]}`;
|
|
23
|
+
}).join(delim);
|
|
24
|
+
}
|
|
25
|
+
function normalizeSelectorValue(v) {
|
|
26
|
+
return v.replace(/[^A-Za-z0-9]+/g, "-");
|
|
27
|
+
}
|
|
28
|
+
function getJobName(name, selectorFlags, mode) {
|
|
29
|
+
const selectorNamePart = Object.keys(selectorFlags).sort().map((name2) => selectorFlags[name2]).join("-");
|
|
30
|
+
let jobName = name;
|
|
31
|
+
if (mode) {
|
|
32
|
+
jobName += `-${mode}`;
|
|
33
|
+
}
|
|
34
|
+
if (selectorNamePart.length > 0) {
|
|
35
|
+
jobName += `-${normalizeSelectorValue(selectorNamePart)}`;
|
|
36
|
+
}
|
|
37
|
+
return jobName;
|
|
38
|
+
}
|
|
1
39
|
// src/env.ts
|
|
2
40
|
function ensureAndGet(envName, defaultValue) {
|
|
3
41
|
return process.env[envName] || defaultValue;
|
|
@@ -17,56 +55,207 @@ function isProduction() {
|
|
|
17
55
|
function isDev() {
|
|
18
56
|
return getEnvironment() === "dev";
|
|
19
57
|
}
|
|
20
|
-
// src/
|
|
21
|
-
function
|
|
22
|
-
|
|
23
|
-
for (let i = startAt;i <= endAt; i += step) {
|
|
24
|
-
arr.push([i, Math.min(endAt, i + step - 1)]);
|
|
25
|
-
}
|
|
26
|
-
return arr;
|
|
58
|
+
// src/log.ts
|
|
59
|
+
function isObject(a) {
|
|
60
|
+
return !!a && a.constructor === Object;
|
|
27
61
|
}
|
|
28
|
-
function
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
groups.push(array.slice(i, i + groupSize));
|
|
62
|
+
function print(o) {
|
|
63
|
+
if (Array.isArray(o)) {
|
|
64
|
+
return `[${o.map(print).join(", ")}]`;
|
|
32
65
|
}
|
|
33
|
-
|
|
66
|
+
if (isObject(o)) {
|
|
67
|
+
return `{${Object.keys(o).map((k) => `${k}: ${o[k]}`).join(", ")}}`;
|
|
68
|
+
}
|
|
69
|
+
return `${o}`;
|
|
34
70
|
}
|
|
35
|
-
function
|
|
36
|
-
|
|
37
|
-
for (let i =
|
|
38
|
-
|
|
71
|
+
function getLine(params) {
|
|
72
|
+
let line = "";
|
|
73
|
+
for (let i = 0, l = params.length;i < l; i++) {
|
|
74
|
+
line += `${print(params[i])} `.replace(/\n/gm, "\t");
|
|
39
75
|
}
|
|
40
|
-
return
|
|
76
|
+
return line.trim();
|
|
41
77
|
}
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
function getDateOnly(date) {
|
|
45
|
-
return new Date(date).toISOString().split("T")[0];
|
|
78
|
+
function timestamp() {
|
|
79
|
+
return new Date().toISOString();
|
|
46
80
|
}
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
81
|
+
var inline = {
|
|
82
|
+
debug: function(...args) {
|
|
83
|
+
if (true) {
|
|
84
|
+
console.log(`${timestamp()} ${getLine(args)}`);
|
|
85
|
+
}
|
|
86
|
+
},
|
|
87
|
+
log: function(...args) {
|
|
88
|
+
if (true) {
|
|
89
|
+
console.log(`${timestamp()} ${getLine(args)}`);
|
|
90
|
+
}
|
|
91
|
+
},
|
|
92
|
+
error: function(...args) {
|
|
93
|
+
if (true) {
|
|
94
|
+
console.error(`${timestamp()} ${getLine(args)}`);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
};
|
|
98
|
+
var logger = {
|
|
99
|
+
debug: function(...args) {
|
|
100
|
+
if (true) {
|
|
101
|
+
console.log(`[${timestamp()}]`, ...args);
|
|
102
|
+
}
|
|
103
|
+
},
|
|
104
|
+
log: function(...args) {
|
|
105
|
+
if (true) {
|
|
106
|
+
console.log(`[${timestamp()}]`, ...args);
|
|
107
|
+
}
|
|
108
|
+
},
|
|
109
|
+
error: function(...args) {
|
|
110
|
+
if (true) {
|
|
111
|
+
console.error(`[${timestamp()}]`, ...args);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
};
|
|
115
|
+
// src/api.ts
|
|
116
|
+
import osModule from "os";
|
|
117
|
+
import express from "express";
|
|
118
|
+
import meow from "meow";
|
|
119
|
+
async function logStartMiddleware(_, res, next) {
|
|
120
|
+
const start = new Date;
|
|
121
|
+
res.set("x-requested-at", start.toISOString());
|
|
122
|
+
next();
|
|
51
123
|
}
|
|
52
|
-
function
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
124
|
+
async function contextMiddleware(_, res, next) {
|
|
125
|
+
res.set("x-instance-id", osModule.hostname());
|
|
126
|
+
next();
|
|
127
|
+
}
|
|
128
|
+
async function logEndMiddleware(req, res, next) {
|
|
129
|
+
const requestedAt = res.get("x-requested-at");
|
|
130
|
+
if (!requestedAt) {
|
|
131
|
+
inline.log("missing x-requested-at header");
|
|
132
|
+
next();
|
|
133
|
+
return;
|
|
57
134
|
}
|
|
58
|
-
const
|
|
59
|
-
const
|
|
60
|
-
|
|
61
|
-
|
|
135
|
+
const start = new Date(requestedAt).getTime();
|
|
136
|
+
const end = new Date().getTime();
|
|
137
|
+
const logInfo = {
|
|
138
|
+
start,
|
|
139
|
+
end,
|
|
140
|
+
elapsed: `${end - start}ms`,
|
|
141
|
+
endpoint: req.path,
|
|
142
|
+
host: req.hostname,
|
|
143
|
+
status: res.statusCode
|
|
144
|
+
};
|
|
145
|
+
if (res.statusMessage) {
|
|
146
|
+
logInfo.errorMessage = res.statusMessage;
|
|
62
147
|
}
|
|
63
|
-
|
|
148
|
+
inline.log(JSON.stringify(logInfo));
|
|
149
|
+
next();
|
|
64
150
|
}
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
151
|
+
var apiKeyMiddleware = (key) => {
|
|
152
|
+
async function requireAPIKey(req, res, next) {
|
|
153
|
+
try {
|
|
154
|
+
const apiKey = req.get("x-api-key") || req.query["api-key"];
|
|
155
|
+
if (!apiKey) {
|
|
156
|
+
inline.log("request without api key");
|
|
157
|
+
res.status(400).send("require x-api-key header");
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
if (typeof key === "string") {
|
|
161
|
+
if (apiKey !== key) {
|
|
162
|
+
inline.log("request has an invalid api key");
|
|
163
|
+
res.status(400).send("invalid api key");
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
inline.log(`requested by valid key ${key.slice(0, 6)}`);
|
|
167
|
+
} else {
|
|
168
|
+
const name = key[apiKey];
|
|
169
|
+
if (!name) {
|
|
170
|
+
inline.log("request has an invalid api key");
|
|
171
|
+
res.status(400).send("invalid api key");
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
inline.log(`requested by authorized user ${name}`);
|
|
175
|
+
}
|
|
176
|
+
next();
|
|
177
|
+
} catch (err) {
|
|
178
|
+
inline.log("check api key error", err);
|
|
179
|
+
res.status(500).send("internal error");
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
return requireAPIKey;
|
|
183
|
+
};
|
|
184
|
+
async function startApiApp({
|
|
185
|
+
binaryName,
|
|
186
|
+
name,
|
|
187
|
+
selector = {},
|
|
188
|
+
routes,
|
|
189
|
+
serve,
|
|
190
|
+
beforeListen
|
|
191
|
+
}) {
|
|
192
|
+
const app = express();
|
|
193
|
+
app.use(express.json({ limit: "20mb" }));
|
|
194
|
+
const cli = meow(`
|
|
195
|
+
Usage
|
|
196
|
+
$ ${binaryName} <options>
|
|
197
|
+
|
|
198
|
+
Options
|
|
199
|
+
${getSelectorDesc(selector)}
|
|
200
|
+
--verbose Output debug messages
|
|
201
|
+
`, {
|
|
202
|
+
importMeta: import.meta,
|
|
203
|
+
description: false,
|
|
204
|
+
flags: {
|
|
205
|
+
...getSelectorFlags(selector),
|
|
206
|
+
verbose: {
|
|
207
|
+
type: "boolean",
|
|
208
|
+
default: false
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
});
|
|
212
|
+
const { verbose, ...selectorFlags } = cli.flags;
|
|
213
|
+
for (const route of routes) {
|
|
214
|
+
const method = route.method ? route.method.toLowerCase() : "get";
|
|
215
|
+
const middlewares = route.middlewares || [];
|
|
216
|
+
if (route.protected) {
|
|
217
|
+
if (!serve.apiKey) {
|
|
218
|
+
throw new Error("serve.apiKey is required for protected route");
|
|
219
|
+
}
|
|
220
|
+
middlewares.unshift(apiKeyMiddleware(serve.apiKey));
|
|
221
|
+
}
|
|
222
|
+
if (app[method]) {
|
|
223
|
+
if (verbose) {
|
|
224
|
+
inline.log(`registering ${method} ${route.path}`);
|
|
225
|
+
}
|
|
226
|
+
app[method](route.path, contextMiddleware, logStartMiddleware, ...middlewares, async (req, res, next) => {
|
|
227
|
+
try {
|
|
228
|
+
await route.handler({ req, res, ...selectorFlags });
|
|
229
|
+
} catch (routeErr) {
|
|
230
|
+
if (routeErr instanceof Error) {
|
|
231
|
+
inline.log("caught route err", routeErr, routeErr.stack);
|
|
232
|
+
res.status(500).send(`internal server error: ${routeErr.message}`);
|
|
233
|
+
} else {
|
|
234
|
+
inline.log("caught route err", routeErr);
|
|
235
|
+
res.status(500).send("internal server error");
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
next();
|
|
239
|
+
}, logEndMiddleware);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
if (!routes.some((r) => r.path === "/" && r.method?.toUpperCase() === "GET")) {
|
|
243
|
+
app.get("/", (_, res) => {
|
|
244
|
+
res.send("ok");
|
|
245
|
+
});
|
|
246
|
+
}
|
|
247
|
+
if (beforeListen) {
|
|
248
|
+
await beforeListen({ app });
|
|
249
|
+
}
|
|
250
|
+
app.listen(serve.port, () => {
|
|
251
|
+
if (isProduction()) {
|
|
252
|
+
inline.log(`${name} listening at https://api.wf.corp.certik.com${serve.prefix}`);
|
|
253
|
+
} else {
|
|
254
|
+
inline.log(`${name} listening at http://localhost:${serve.port}`);
|
|
255
|
+
}
|
|
256
|
+
});
|
|
69
257
|
}
|
|
258
|
+
|
|
70
259
|
// src/object-hash.ts
|
|
71
260
|
import xh from "@node-rs/xxhash";
|
|
72
261
|
function getHash(obj) {
|
|
@@ -191,65 +380,30 @@ function memoize(func, options) {
|
|
|
191
380
|
if (!options.cacheKey) {
|
|
192
381
|
options.cacheKey = (args) => getHash(args);
|
|
193
382
|
}
|
|
194
|
-
return pMemoize(func, options);
|
|
195
|
-
}
|
|
196
|
-
// src/
|
|
197
|
-
function
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
if (Array.isArray(o)) {
|
|
202
|
-
return `[${o.map(print).join(", ")}]`;
|
|
203
|
-
}
|
|
204
|
-
if (isObject(o)) {
|
|
205
|
-
return `{${Object.keys(o).map((k) => `${k}: ${o[k]}`).join(", ")}}`;
|
|
206
|
-
}
|
|
207
|
-
return `${o}`;
|
|
208
|
-
}
|
|
209
|
-
function getLine(params) {
|
|
210
|
-
let line = "";
|
|
211
|
-
for (let i = 0, l = params.length;i < l; i++) {
|
|
212
|
-
line += `${print(params[i])} `.replace(/\n/gm, "\t");
|
|
213
|
-
}
|
|
214
|
-
return line.trim();
|
|
215
|
-
}
|
|
216
|
-
function timestamp() {
|
|
217
|
-
return new Date().toISOString();
|
|
218
|
-
}
|
|
219
|
-
var inline = {
|
|
220
|
-
debug: function(...args) {
|
|
221
|
-
if (true) {
|
|
222
|
-
console.log(`${timestamp()} ${getLine(args)}`);
|
|
223
|
-
}
|
|
224
|
-
},
|
|
225
|
-
log: function(...args) {
|
|
226
|
-
if (true) {
|
|
227
|
-
console.log(`${timestamp()} ${getLine(args)}`);
|
|
228
|
-
}
|
|
229
|
-
},
|
|
230
|
-
error: function(...args) {
|
|
231
|
-
if (true) {
|
|
232
|
-
console.error(`${timestamp()} ${getLine(args)}`);
|
|
233
|
-
}
|
|
383
|
+
return pMemoize(func, options);
|
|
384
|
+
}
|
|
385
|
+
// src/util.ts
|
|
386
|
+
function range(startAt, endAt, step) {
|
|
387
|
+
const arr = [];
|
|
388
|
+
for (let i = startAt;i <= endAt; i += step) {
|
|
389
|
+
arr.push([i, Math.min(endAt, i + step - 1)]);
|
|
234
390
|
}
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
},
|
|
242
|
-
log: function(...args) {
|
|
243
|
-
if (true) {
|
|
244
|
-
console.log(`[${timestamp()}]`, ...args);
|
|
245
|
-
}
|
|
246
|
-
},
|
|
247
|
-
error: function(...args) {
|
|
248
|
-
if (true) {
|
|
249
|
-
console.error(`[${timestamp()}]`, ...args);
|
|
250
|
-
}
|
|
391
|
+
return arr;
|
|
392
|
+
}
|
|
393
|
+
function arrayGroup(array, groupSize) {
|
|
394
|
+
const groups = [];
|
|
395
|
+
for (let i = 0;i < array.length; i += groupSize) {
|
|
396
|
+
groups.push(array.slice(i, i + groupSize));
|
|
251
397
|
}
|
|
252
|
-
|
|
398
|
+
return groups;
|
|
399
|
+
}
|
|
400
|
+
function fillRange(start, end) {
|
|
401
|
+
const result = [];
|
|
402
|
+
for (let i = start;i <= end; i++) {
|
|
403
|
+
result.push(i);
|
|
404
|
+
}
|
|
405
|
+
return result;
|
|
406
|
+
}
|
|
253
407
|
// src/dynamodb.ts
|
|
254
408
|
import {
|
|
255
409
|
DynamoDBDocumentClient,
|
|
@@ -564,44 +718,6 @@ async function deleteRecordsByHashKey(tableName, indexName, hashKeyValue, verbos
|
|
|
564
718
|
}
|
|
565
719
|
return totalDeleted;
|
|
566
720
|
}
|
|
567
|
-
// src/selector.ts
|
|
568
|
-
function getSelectorDesc(selector) {
|
|
569
|
-
return Object.keys(selector).map((name) => {
|
|
570
|
-
return ` --${name.padEnd(14)}${selector[name].desc || selector[name].description || ""}`;
|
|
571
|
-
}).join(`
|
|
572
|
-
`);
|
|
573
|
-
}
|
|
574
|
-
function getSelectorFlags(selector) {
|
|
575
|
-
return Object.keys(selector).reduce((acc, name) => {
|
|
576
|
-
const flag = {
|
|
577
|
-
type: selector[name].type || "string",
|
|
578
|
-
...selector[name]
|
|
579
|
-
};
|
|
580
|
-
if (!selector[name].optional && selector[name].isRequired !== false) {
|
|
581
|
-
flag.isRequired = true;
|
|
582
|
-
}
|
|
583
|
-
return { ...acc, [name]: flag };
|
|
584
|
-
}, {});
|
|
585
|
-
}
|
|
586
|
-
function toSelectorString(selectorFlags, delim = ",") {
|
|
587
|
-
return Object.keys(selectorFlags).sort().map((flag) => {
|
|
588
|
-
return `${flag}=${selectorFlags[flag]}`;
|
|
589
|
-
}).join(delim);
|
|
590
|
-
}
|
|
591
|
-
function normalizeSelectorValue(v) {
|
|
592
|
-
return v.replace(/[^A-Za-z0-9]+/g, "-");
|
|
593
|
-
}
|
|
594
|
-
function getJobName(name, selectorFlags, mode) {
|
|
595
|
-
const selectorNamePart = Object.keys(selectorFlags).sort().map((name2) => selectorFlags[name2]).join("-");
|
|
596
|
-
let jobName = name;
|
|
597
|
-
if (mode) {
|
|
598
|
-
jobName += `-${mode}`;
|
|
599
|
-
}
|
|
600
|
-
if (selectorNamePart.length > 0) {
|
|
601
|
-
jobName += `-${normalizeSelectorValue(selectorNamePart)}`;
|
|
602
|
-
}
|
|
603
|
-
return jobName;
|
|
604
|
-
}
|
|
605
721
|
// src/cli.ts
|
|
606
722
|
import path from "path";
|
|
607
723
|
import fs from "fs";
|
|
@@ -637,8 +753,36 @@ function detectBin() {
|
|
|
637
753
|
const wd = detectDirectory(process.argv[1], "package.json");
|
|
638
754
|
return process.argv[1].slice(wd.length + path.sep.length).replace(path.sep, "/");
|
|
639
755
|
}
|
|
756
|
+
// src/date.ts
|
|
757
|
+
var MS_IN_A_DAY = 3600 * 24 * 1000;
|
|
758
|
+
function getDateOnly(date) {
|
|
759
|
+
return new Date(date).toISOString().split("T")[0];
|
|
760
|
+
}
|
|
761
|
+
function findDateAfter(date, n) {
|
|
762
|
+
const d = new Date(date);
|
|
763
|
+
const after = new Date(d.getTime() + MS_IN_A_DAY * n);
|
|
764
|
+
return getDateOnly(after);
|
|
765
|
+
}
|
|
766
|
+
function daysInRange(from, to) {
|
|
767
|
+
const fromTime = new Date(from).getTime();
|
|
768
|
+
const toTime = new Date(to).getTime();
|
|
769
|
+
if (fromTime > toTime) {
|
|
770
|
+
throw new Error(`range to date couldn't be earlier than range from date`);
|
|
771
|
+
}
|
|
772
|
+
const daysBetween = Math.floor((toTime - fromTime) / MS_IN_A_DAY);
|
|
773
|
+
const dates = [getDateOnly(new Date(fromTime))];
|
|
774
|
+
for (let i = 1;i <= daysBetween; i += 1) {
|
|
775
|
+
dates.push(getDateOnly(new Date(fromTime + i * MS_IN_A_DAY)));
|
|
776
|
+
}
|
|
777
|
+
return dates;
|
|
778
|
+
}
|
|
779
|
+
function dateRange(from, to, step) {
|
|
780
|
+
const days = daysInRange(from, to);
|
|
781
|
+
const windows = arrayGroup(days, step);
|
|
782
|
+
return windows.map((w) => [w[0], w[w.length - 1]]);
|
|
783
|
+
}
|
|
640
784
|
// src/indexer.ts
|
|
641
|
-
import
|
|
785
|
+
import meow2 from "meow";
|
|
642
786
|
var STATE_TABLE_NAME = "skynet-" + getEnvironment() + "-indexer-state";
|
|
643
787
|
async function getIndexerLatestId(name, selectorFlags) {
|
|
644
788
|
const record = await getRecordByKey(STATE_TABLE_NAME, {
|
|
@@ -981,7 +1125,7 @@ function createModeIndexerApp({
|
|
|
981
1125
|
if (!binaryName) {
|
|
982
1126
|
binaryName = getBinaryName();
|
|
983
1127
|
}
|
|
984
|
-
const cli =
|
|
1128
|
+
const cli = meow2(`
|
|
985
1129
|
Usage
|
|
986
1130
|
|
|
987
1131
|
$ ${binaryName} <options>
|
|
@@ -1039,7 +1183,7 @@ function createIndexerApp({
|
|
|
1039
1183
|
if (!binaryName) {
|
|
1040
1184
|
binaryName = getBinaryName();
|
|
1041
1185
|
}
|
|
1042
|
-
const cli =
|
|
1186
|
+
const cli = meow2(`
|
|
1043
1187
|
Usage
|
|
1044
1188
|
$ ${binaryName} <options>
|
|
1045
1189
|
|
|
@@ -1096,7 +1240,7 @@ ${selector ? getSelectorDesc(selector) : ""}
|
|
|
1096
1240
|
import fs2 from "fs/promises";
|
|
1097
1241
|
import fso from "fs";
|
|
1098
1242
|
import { execa } from "execa";
|
|
1099
|
-
import
|
|
1243
|
+
import meow3 from "meow";
|
|
1100
1244
|
import chalk from "chalk";
|
|
1101
1245
|
import which from "which";
|
|
1102
1246
|
var INTERVAL_ALIASES = {
|
|
@@ -1174,7 +1318,7 @@ var genConfig = ({
|
|
|
1174
1318
|
command = "sh"
|
|
1175
1319
|
args = [
|
|
1176
1320
|
"-c",
|
|
1177
|
-
"cd \${meta.skynet_code_path}/${workingDirectory} && if [ -e bun.lockb ]; then bun install --silent; else yarn install --silent; fi && exec ${cmd}"
|
|
1321
|
+
"cd \${meta.skynet_code_path}/${workingDirectory} && if [ -e bun.lockb ] || [ -e bun.lock ]; then bun install --production --silent; else yarn install --production --silent; fi && exec ${cmd}"
|
|
1178
1322
|
]
|
|
1179
1323
|
}
|
|
1180
1324
|
|
|
@@ -1403,7 +1547,7 @@ function createModeDeploy({
|
|
|
1403
1547
|
if (!binaryName) {
|
|
1404
1548
|
binaryName = getBinaryName();
|
|
1405
1549
|
}
|
|
1406
|
-
const cli =
|
|
1550
|
+
const cli = meow3(`
|
|
1407
1551
|
Usage
|
|
1408
1552
|
|
|
1409
1553
|
$ ${binaryName} <options>
|
|
@@ -1533,7 +1677,7 @@ function createDeploy({
|
|
|
1533
1677
|
if (!binaryName) {
|
|
1534
1678
|
binaryName = getBinaryName();
|
|
1535
1679
|
}
|
|
1536
|
-
const cli =
|
|
1680
|
+
const cli = meow3(`
|
|
1537
1681
|
Usage
|
|
1538
1682
|
|
|
1539
1683
|
$ ${binaryName} <options>
|
|
@@ -1581,150 +1725,6 @@ ${getSelectorDesc(selector)}
|
|
|
1581
1725
|
}
|
|
1582
1726
|
return { deploy };
|
|
1583
1727
|
}
|
|
1584
|
-
// src/api.ts
|
|
1585
|
-
import osModule from "os";
|
|
1586
|
-
import express from "express";
|
|
1587
|
-
import meow3 from "meow";
|
|
1588
|
-
async function logStartMiddleware(_, res, next) {
|
|
1589
|
-
const start = new Date;
|
|
1590
|
-
res.set("x-requested-at", start.toISOString());
|
|
1591
|
-
next();
|
|
1592
|
-
}
|
|
1593
|
-
async function contextMiddleware(_, res, next) {
|
|
1594
|
-
res.set("x-instance-id", osModule.hostname());
|
|
1595
|
-
next();
|
|
1596
|
-
}
|
|
1597
|
-
async function logEndMiddleware(req, res, next) {
|
|
1598
|
-
const requestedAt = res.get("x-requested-at");
|
|
1599
|
-
if (!requestedAt) {
|
|
1600
|
-
inline.log("missing x-requested-at header");
|
|
1601
|
-
next();
|
|
1602
|
-
return;
|
|
1603
|
-
}
|
|
1604
|
-
const start = new Date(requestedAt).getTime();
|
|
1605
|
-
const end = new Date().getTime();
|
|
1606
|
-
const logInfo = {
|
|
1607
|
-
start,
|
|
1608
|
-
end,
|
|
1609
|
-
elapsed: `${end - start}ms`,
|
|
1610
|
-
endpoint: req.path,
|
|
1611
|
-
host: req.hostname,
|
|
1612
|
-
status: res.statusCode
|
|
1613
|
-
};
|
|
1614
|
-
if (res.statusMessage) {
|
|
1615
|
-
logInfo.errorMessage = res.statusMessage;
|
|
1616
|
-
}
|
|
1617
|
-
inline.log(JSON.stringify(logInfo));
|
|
1618
|
-
next();
|
|
1619
|
-
}
|
|
1620
|
-
var apiKeyMiddleware = (key) => {
|
|
1621
|
-
async function requireAPIKey(req, res, next) {
|
|
1622
|
-
try {
|
|
1623
|
-
const apiKey = req.get("x-api-key") || req.query["api-key"];
|
|
1624
|
-
if (!apiKey) {
|
|
1625
|
-
inline.log("request without api key");
|
|
1626
|
-
res.status(400).send("require x-api-key header");
|
|
1627
|
-
return;
|
|
1628
|
-
}
|
|
1629
|
-
if (typeof key === "string") {
|
|
1630
|
-
if (apiKey !== key) {
|
|
1631
|
-
inline.log("request has an invalid api key");
|
|
1632
|
-
res.status(400).send("invalid api key");
|
|
1633
|
-
return;
|
|
1634
|
-
}
|
|
1635
|
-
inline.log(`requested by valid key ${key.slice(0, 6)}`);
|
|
1636
|
-
} else {
|
|
1637
|
-
const name = key[apiKey];
|
|
1638
|
-
if (!name) {
|
|
1639
|
-
inline.log("request has an invalid api key");
|
|
1640
|
-
res.status(400).send("invalid api key");
|
|
1641
|
-
return;
|
|
1642
|
-
}
|
|
1643
|
-
inline.log(`requested by authorized user ${name}`);
|
|
1644
|
-
}
|
|
1645
|
-
next();
|
|
1646
|
-
} catch (err) {
|
|
1647
|
-
inline.log("check api key error", err);
|
|
1648
|
-
res.status(500).send("internal error");
|
|
1649
|
-
}
|
|
1650
|
-
}
|
|
1651
|
-
return requireAPIKey;
|
|
1652
|
-
};
|
|
1653
|
-
async function startApiApp({
|
|
1654
|
-
binaryName,
|
|
1655
|
-
name,
|
|
1656
|
-
selector = {},
|
|
1657
|
-
routes,
|
|
1658
|
-
serve,
|
|
1659
|
-
beforeListen
|
|
1660
|
-
}) {
|
|
1661
|
-
const app = express();
|
|
1662
|
-
app.use(express.json({ limit: "20mb" }));
|
|
1663
|
-
const cli = meow3(`
|
|
1664
|
-
Usage
|
|
1665
|
-
$ ${binaryName} <options>
|
|
1666
|
-
|
|
1667
|
-
Options
|
|
1668
|
-
${getSelectorDesc(selector)}
|
|
1669
|
-
--verbose Output debug messages
|
|
1670
|
-
`, {
|
|
1671
|
-
importMeta: import.meta,
|
|
1672
|
-
description: false,
|
|
1673
|
-
flags: {
|
|
1674
|
-
...getSelectorFlags(selector),
|
|
1675
|
-
verbose: {
|
|
1676
|
-
type: "boolean",
|
|
1677
|
-
default: false
|
|
1678
|
-
}
|
|
1679
|
-
}
|
|
1680
|
-
});
|
|
1681
|
-
const { verbose, ...selectorFlags } = cli.flags;
|
|
1682
|
-
for (const route of routes) {
|
|
1683
|
-
const method = route.method ? route.method.toLowerCase() : "get";
|
|
1684
|
-
const middlewares = route.middlewares || [];
|
|
1685
|
-
if (route.protected) {
|
|
1686
|
-
if (!serve.apiKey) {
|
|
1687
|
-
throw new Error("serve.apiKey is required for protected route");
|
|
1688
|
-
}
|
|
1689
|
-
middlewares.unshift(apiKeyMiddleware(serve.apiKey));
|
|
1690
|
-
}
|
|
1691
|
-
if (app[method]) {
|
|
1692
|
-
if (verbose) {
|
|
1693
|
-
inline.log(`registering ${method} ${route.path}`);
|
|
1694
|
-
}
|
|
1695
|
-
app[method](route.path, contextMiddleware, logStartMiddleware, ...middlewares, async (req, res, next) => {
|
|
1696
|
-
try {
|
|
1697
|
-
await route.handler({ req, res, ...selectorFlags });
|
|
1698
|
-
} catch (routeErr) {
|
|
1699
|
-
if (routeErr instanceof Error) {
|
|
1700
|
-
inline.log("caught route err", routeErr, routeErr.stack);
|
|
1701
|
-
res.status(500).send(`internal server error: ${routeErr.message}`);
|
|
1702
|
-
} else {
|
|
1703
|
-
inline.log("caught route err", routeErr);
|
|
1704
|
-
res.status(500).send("internal server error");
|
|
1705
|
-
}
|
|
1706
|
-
}
|
|
1707
|
-
next();
|
|
1708
|
-
}, logEndMiddleware);
|
|
1709
|
-
}
|
|
1710
|
-
}
|
|
1711
|
-
if (!routes.some((r) => r.path === "/" && r.method?.toUpperCase() === "GET")) {
|
|
1712
|
-
app.get("/", (_, res) => {
|
|
1713
|
-
res.send("ok");
|
|
1714
|
-
});
|
|
1715
|
-
}
|
|
1716
|
-
if (beforeListen) {
|
|
1717
|
-
await beforeListen({ app });
|
|
1718
|
-
}
|
|
1719
|
-
app.listen(serve.port, () => {
|
|
1720
|
-
if (isProduction()) {
|
|
1721
|
-
inline.log(`${name} listening at https://api.wf.corp.certik.com${serve.prefix}`);
|
|
1722
|
-
} else {
|
|
1723
|
-
inline.log(`${name} listening at http://localhost:${serve.port}`);
|
|
1724
|
-
}
|
|
1725
|
-
});
|
|
1726
|
-
}
|
|
1727
|
-
|
|
1728
1728
|
// src/app.ts
|
|
1729
1729
|
import { EOL } from "os";
|
|
1730
1730
|
function printAppHelp() {
|
package/dist/deploy.js
CHANGED
|
@@ -1,22 +1,3 @@
|
|
|
1
|
-
// src/env.ts
|
|
2
|
-
function ensureAndGet(envName, defaultValue) {
|
|
3
|
-
return process.env[envName] || defaultValue;
|
|
4
|
-
}
|
|
5
|
-
function getEnvironment() {
|
|
6
|
-
return ensureAndGet("SKYNET_ENVIRONMENT", "dev");
|
|
7
|
-
}
|
|
8
|
-
function getEnvOrThrow(envName) {
|
|
9
|
-
if (!process.env[envName]) {
|
|
10
|
-
throw new Error(`Must set environment variable ${envName}`);
|
|
11
|
-
}
|
|
12
|
-
return process.env[envName];
|
|
13
|
-
}
|
|
14
|
-
function isProduction() {
|
|
15
|
-
return getEnvironment() === "prd";
|
|
16
|
-
}
|
|
17
|
-
function isDev() {
|
|
18
|
-
return getEnvironment() === "dev";
|
|
19
|
-
}
|
|
20
1
|
// src/selector.ts
|
|
21
2
|
function getSelectorDesc(selector) {
|
|
22
3
|
return Object.keys(selector).map((name) => {
|
|
@@ -55,6 +36,25 @@ function getJobName(name, selectorFlags, mode) {
|
|
|
55
36
|
}
|
|
56
37
|
return jobName;
|
|
57
38
|
}
|
|
39
|
+
// src/env.ts
|
|
40
|
+
function ensureAndGet(envName, defaultValue) {
|
|
41
|
+
return process.env[envName] || defaultValue;
|
|
42
|
+
}
|
|
43
|
+
function getEnvironment() {
|
|
44
|
+
return ensureAndGet("SKYNET_ENVIRONMENT", "dev");
|
|
45
|
+
}
|
|
46
|
+
function getEnvOrThrow(envName) {
|
|
47
|
+
if (!process.env[envName]) {
|
|
48
|
+
throw new Error(`Must set environment variable ${envName}`);
|
|
49
|
+
}
|
|
50
|
+
return process.env[envName];
|
|
51
|
+
}
|
|
52
|
+
function isProduction() {
|
|
53
|
+
return getEnvironment() === "prd";
|
|
54
|
+
}
|
|
55
|
+
function isDev() {
|
|
56
|
+
return getEnvironment() === "dev";
|
|
57
|
+
}
|
|
58
58
|
// src/cli.ts
|
|
59
59
|
import path from "path";
|
|
60
60
|
import fs from "fs";
|
|
@@ -172,7 +172,7 @@ var genConfig = ({
|
|
|
172
172
|
command = "sh"
|
|
173
173
|
args = [
|
|
174
174
|
"-c",
|
|
175
|
-
"cd \${meta.skynet_code_path}/${workingDirectory} && if [ -e bun.lockb ]; then bun install --silent; else yarn install --silent; fi && exec ${cmd}"
|
|
175
|
+
"cd \${meta.skynet_code_path}/${workingDirectory} && if [ -e bun.lockb ] || [ -e bun.lock ]; then bun install --production --silent; else yarn install --production --silent; fi && exec ${cmd}"
|
|
176
176
|
]
|
|
177
177
|
}
|
|
178
178
|
|
package/dist/dynamodb.js
CHANGED
|
@@ -1,25 +1,3 @@
|
|
|
1
|
-
// src/util.ts
|
|
2
|
-
function range(startAt, endAt, step) {
|
|
3
|
-
const arr = [];
|
|
4
|
-
for (let i = startAt;i <= endAt; i += step) {
|
|
5
|
-
arr.push([i, Math.min(endAt, i + step - 1)]);
|
|
6
|
-
}
|
|
7
|
-
return arr;
|
|
8
|
-
}
|
|
9
|
-
function arrayGroup(array, groupSize) {
|
|
10
|
-
const groups = [];
|
|
11
|
-
for (let i = 0;i < array.length; i += groupSize) {
|
|
12
|
-
groups.push(array.slice(i, i + groupSize));
|
|
13
|
-
}
|
|
14
|
-
return groups;
|
|
15
|
-
}
|
|
16
|
-
function fillRange(start, end) {
|
|
17
|
-
const result = [];
|
|
18
|
-
for (let i = start;i <= end; i++) {
|
|
19
|
-
result.push(i);
|
|
20
|
-
}
|
|
21
|
-
return result;
|
|
22
|
-
}
|
|
23
1
|
// src/object-hash.ts
|
|
24
2
|
import xh from "@node-rs/xxhash";
|
|
25
3
|
function getHash(obj) {
|
|
@@ -146,6 +124,28 @@ function memoize(func, options) {
|
|
|
146
124
|
}
|
|
147
125
|
return pMemoize(func, options);
|
|
148
126
|
}
|
|
127
|
+
// src/util.ts
|
|
128
|
+
function range(startAt, endAt, step) {
|
|
129
|
+
const arr = [];
|
|
130
|
+
for (let i = startAt;i <= endAt; i += step) {
|
|
131
|
+
arr.push([i, Math.min(endAt, i + step - 1)]);
|
|
132
|
+
}
|
|
133
|
+
return arr;
|
|
134
|
+
}
|
|
135
|
+
function arrayGroup(array, groupSize) {
|
|
136
|
+
const groups = [];
|
|
137
|
+
for (let i = 0;i < array.length; i += groupSize) {
|
|
138
|
+
groups.push(array.slice(i, i + groupSize));
|
|
139
|
+
}
|
|
140
|
+
return groups;
|
|
141
|
+
}
|
|
142
|
+
function fillRange(start, end) {
|
|
143
|
+
const result = [];
|
|
144
|
+
for (let i = start;i <= end; i++) {
|
|
145
|
+
result.push(i);
|
|
146
|
+
}
|
|
147
|
+
return result;
|
|
148
|
+
}
|
|
149
149
|
// src/dynamodb.ts
|
|
150
150
|
import {
|
|
151
151
|
DynamoDBDocumentClient,
|
package/dist/indexer.js
CHANGED
|
@@ -1,3 +1,41 @@
|
|
|
1
|
+
// src/selector.ts
|
|
2
|
+
function getSelectorDesc(selector) {
|
|
3
|
+
return Object.keys(selector).map((name) => {
|
|
4
|
+
return ` --${name.padEnd(14)}${selector[name].desc || selector[name].description || ""}`;
|
|
5
|
+
}).join(`
|
|
6
|
+
`);
|
|
7
|
+
}
|
|
8
|
+
function getSelectorFlags(selector) {
|
|
9
|
+
return Object.keys(selector).reduce((acc, name) => {
|
|
10
|
+
const flag = {
|
|
11
|
+
type: selector[name].type || "string",
|
|
12
|
+
...selector[name]
|
|
13
|
+
};
|
|
14
|
+
if (!selector[name].optional && selector[name].isRequired !== false) {
|
|
15
|
+
flag.isRequired = true;
|
|
16
|
+
}
|
|
17
|
+
return { ...acc, [name]: flag };
|
|
18
|
+
}, {});
|
|
19
|
+
}
|
|
20
|
+
function toSelectorString(selectorFlags, delim = ",") {
|
|
21
|
+
return Object.keys(selectorFlags).sort().map((flag) => {
|
|
22
|
+
return `${flag}=${selectorFlags[flag]}`;
|
|
23
|
+
}).join(delim);
|
|
24
|
+
}
|
|
25
|
+
function normalizeSelectorValue(v) {
|
|
26
|
+
return v.replace(/[^A-Za-z0-9]+/g, "-");
|
|
27
|
+
}
|
|
28
|
+
function getJobName(name, selectorFlags, mode) {
|
|
29
|
+
const selectorNamePart = Object.keys(selectorFlags).sort().map((name2) => selectorFlags[name2]).join("-");
|
|
30
|
+
let jobName = name;
|
|
31
|
+
if (mode) {
|
|
32
|
+
jobName += `-${mode}`;
|
|
33
|
+
}
|
|
34
|
+
if (selectorNamePart.length > 0) {
|
|
35
|
+
jobName += `-${normalizeSelectorValue(selectorNamePart)}`;
|
|
36
|
+
}
|
|
37
|
+
return jobName;
|
|
38
|
+
}
|
|
1
39
|
// src/env.ts
|
|
2
40
|
function ensureAndGet(envName, defaultValue) {
|
|
3
41
|
return process.env[envName] || defaultValue;
|
|
@@ -17,56 +55,63 @@ function isProduction() {
|
|
|
17
55
|
function isDev() {
|
|
18
56
|
return getEnvironment() === "dev";
|
|
19
57
|
}
|
|
20
|
-
// src/
|
|
21
|
-
function
|
|
22
|
-
|
|
23
|
-
for (let i = startAt;i <= endAt; i += step) {
|
|
24
|
-
arr.push([i, Math.min(endAt, i + step - 1)]);
|
|
25
|
-
}
|
|
26
|
-
return arr;
|
|
58
|
+
// src/log.ts
|
|
59
|
+
function isObject(a) {
|
|
60
|
+
return !!a && a.constructor === Object;
|
|
27
61
|
}
|
|
28
|
-
function
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
groups.push(array.slice(i, i + groupSize));
|
|
62
|
+
function print(o) {
|
|
63
|
+
if (Array.isArray(o)) {
|
|
64
|
+
return `[${o.map(print).join(", ")}]`;
|
|
32
65
|
}
|
|
33
|
-
|
|
34
|
-
}
|
|
35
|
-
function fillRange(start, end) {
|
|
36
|
-
const result = [];
|
|
37
|
-
for (let i = start;i <= end; i++) {
|
|
38
|
-
result.push(i);
|
|
66
|
+
if (isObject(o)) {
|
|
67
|
+
return `{${Object.keys(o).map((k) => `${k}: ${o[k]}`).join(", ")}}`;
|
|
39
68
|
}
|
|
40
|
-
return
|
|
69
|
+
return `${o}`;
|
|
41
70
|
}
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
71
|
+
function getLine(params) {
|
|
72
|
+
let line = "";
|
|
73
|
+
for (let i = 0, l = params.length;i < l; i++) {
|
|
74
|
+
line += `${print(params[i])} `.replace(/\n/gm, "\t");
|
|
75
|
+
}
|
|
76
|
+
return line.trim();
|
|
46
77
|
}
|
|
47
|
-
function
|
|
48
|
-
|
|
49
|
-
const after = new Date(d.getTime() + MS_IN_A_DAY * n);
|
|
50
|
-
return getDateOnly(after);
|
|
78
|
+
function timestamp() {
|
|
79
|
+
return new Date().toISOString();
|
|
51
80
|
}
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
81
|
+
var inline = {
|
|
82
|
+
debug: function(...args) {
|
|
83
|
+
if (true) {
|
|
84
|
+
console.log(`${timestamp()} ${getLine(args)}`);
|
|
85
|
+
}
|
|
86
|
+
},
|
|
87
|
+
log: function(...args) {
|
|
88
|
+
if (true) {
|
|
89
|
+
console.log(`${timestamp()} ${getLine(args)}`);
|
|
90
|
+
}
|
|
91
|
+
},
|
|
92
|
+
error: function(...args) {
|
|
93
|
+
if (true) {
|
|
94
|
+
console.error(`${timestamp()} ${getLine(args)}`);
|
|
95
|
+
}
|
|
57
96
|
}
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
97
|
+
};
|
|
98
|
+
var logger = {
|
|
99
|
+
debug: function(...args) {
|
|
100
|
+
if (true) {
|
|
101
|
+
console.log(`[${timestamp()}]`, ...args);
|
|
102
|
+
}
|
|
103
|
+
},
|
|
104
|
+
log: function(...args) {
|
|
105
|
+
if (true) {
|
|
106
|
+
console.log(`[${timestamp()}]`, ...args);
|
|
107
|
+
}
|
|
108
|
+
},
|
|
109
|
+
error: function(...args) {
|
|
110
|
+
if (true) {
|
|
111
|
+
console.error(`[${timestamp()}]`, ...args);
|
|
112
|
+
}
|
|
62
113
|
}
|
|
63
|
-
|
|
64
|
-
}
|
|
65
|
-
function dateRange(from, to, step) {
|
|
66
|
-
const days = daysInRange(from, to);
|
|
67
|
-
const windows = arrayGroup(days, step);
|
|
68
|
-
return windows.map((w) => [w[0], w[w.length - 1]]);
|
|
69
|
-
}
|
|
114
|
+
};
|
|
70
115
|
// src/object-hash.ts
|
|
71
116
|
import xh from "@node-rs/xxhash";
|
|
72
117
|
function getHash(obj) {
|
|
@@ -193,63 +238,28 @@ function memoize(func, options) {
|
|
|
193
238
|
}
|
|
194
239
|
return pMemoize(func, options);
|
|
195
240
|
}
|
|
196
|
-
// src/
|
|
197
|
-
function
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
if (Array.isArray(o)) {
|
|
202
|
-
return `[${o.map(print).join(", ")}]`;
|
|
203
|
-
}
|
|
204
|
-
if (isObject(o)) {
|
|
205
|
-
return `{${Object.keys(o).map((k) => `${k}: ${o[k]}`).join(", ")}}`;
|
|
241
|
+
// src/util.ts
|
|
242
|
+
function range(startAt, endAt, step) {
|
|
243
|
+
const arr = [];
|
|
244
|
+
for (let i = startAt;i <= endAt; i += step) {
|
|
245
|
+
arr.push([i, Math.min(endAt, i + step - 1)]);
|
|
206
246
|
}
|
|
207
|
-
return
|
|
247
|
+
return arr;
|
|
208
248
|
}
|
|
209
|
-
function
|
|
210
|
-
|
|
211
|
-
for (let i = 0
|
|
212
|
-
|
|
249
|
+
function arrayGroup(array, groupSize) {
|
|
250
|
+
const groups = [];
|
|
251
|
+
for (let i = 0;i < array.length; i += groupSize) {
|
|
252
|
+
groups.push(array.slice(i, i + groupSize));
|
|
213
253
|
}
|
|
214
|
-
return
|
|
215
|
-
}
|
|
216
|
-
function timestamp() {
|
|
217
|
-
return new Date().toISOString();
|
|
254
|
+
return groups;
|
|
218
255
|
}
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
}
|
|
224
|
-
},
|
|
225
|
-
log: function(...args) {
|
|
226
|
-
if (true) {
|
|
227
|
-
console.log(`${timestamp()} ${getLine(args)}`);
|
|
228
|
-
}
|
|
229
|
-
},
|
|
230
|
-
error: function(...args) {
|
|
231
|
-
if (true) {
|
|
232
|
-
console.error(`${timestamp()} ${getLine(args)}`);
|
|
233
|
-
}
|
|
234
|
-
}
|
|
235
|
-
};
|
|
236
|
-
var logger = {
|
|
237
|
-
debug: function(...args) {
|
|
238
|
-
if (true) {
|
|
239
|
-
console.log(`[${timestamp()}]`, ...args);
|
|
240
|
-
}
|
|
241
|
-
},
|
|
242
|
-
log: function(...args) {
|
|
243
|
-
if (true) {
|
|
244
|
-
console.log(`[${timestamp()}]`, ...args);
|
|
245
|
-
}
|
|
246
|
-
},
|
|
247
|
-
error: function(...args) {
|
|
248
|
-
if (true) {
|
|
249
|
-
console.error(`[${timestamp()}]`, ...args);
|
|
250
|
-
}
|
|
256
|
+
function fillRange(start, end) {
|
|
257
|
+
const result = [];
|
|
258
|
+
for (let i = start;i <= end; i++) {
|
|
259
|
+
result.push(i);
|
|
251
260
|
}
|
|
252
|
-
|
|
261
|
+
return result;
|
|
262
|
+
}
|
|
253
263
|
// src/dynamodb.ts
|
|
254
264
|
import {
|
|
255
265
|
DynamoDBDocumentClient,
|
|
@@ -564,44 +574,6 @@ async function deleteRecordsByHashKey(tableName, indexName, hashKeyValue, verbos
|
|
|
564
574
|
}
|
|
565
575
|
return totalDeleted;
|
|
566
576
|
}
|
|
567
|
-
// src/selector.ts
|
|
568
|
-
function getSelectorDesc(selector) {
|
|
569
|
-
return Object.keys(selector).map((name) => {
|
|
570
|
-
return ` --${name.padEnd(14)}${selector[name].desc || selector[name].description || ""}`;
|
|
571
|
-
}).join(`
|
|
572
|
-
`);
|
|
573
|
-
}
|
|
574
|
-
function getSelectorFlags(selector) {
|
|
575
|
-
return Object.keys(selector).reduce((acc, name) => {
|
|
576
|
-
const flag = {
|
|
577
|
-
type: selector[name].type || "string",
|
|
578
|
-
...selector[name]
|
|
579
|
-
};
|
|
580
|
-
if (!selector[name].optional && selector[name].isRequired !== false) {
|
|
581
|
-
flag.isRequired = true;
|
|
582
|
-
}
|
|
583
|
-
return { ...acc, [name]: flag };
|
|
584
|
-
}, {});
|
|
585
|
-
}
|
|
586
|
-
function toSelectorString(selectorFlags, delim = ",") {
|
|
587
|
-
return Object.keys(selectorFlags).sort().map((flag) => {
|
|
588
|
-
return `${flag}=${selectorFlags[flag]}`;
|
|
589
|
-
}).join(delim);
|
|
590
|
-
}
|
|
591
|
-
function normalizeSelectorValue(v) {
|
|
592
|
-
return v.replace(/[^A-Za-z0-9]+/g, "-");
|
|
593
|
-
}
|
|
594
|
-
function getJobName(name, selectorFlags, mode) {
|
|
595
|
-
const selectorNamePart = Object.keys(selectorFlags).sort().map((name2) => selectorFlags[name2]).join("-");
|
|
596
|
-
let jobName = name;
|
|
597
|
-
if (mode) {
|
|
598
|
-
jobName += `-${mode}`;
|
|
599
|
-
}
|
|
600
|
-
if (selectorNamePart.length > 0) {
|
|
601
|
-
jobName += `-${normalizeSelectorValue(selectorNamePart)}`;
|
|
602
|
-
}
|
|
603
|
-
return jobName;
|
|
604
|
-
}
|
|
605
577
|
// src/cli.ts
|
|
606
578
|
import path from "path";
|
|
607
579
|
import fs from "fs";
|
|
@@ -637,6 +609,34 @@ function detectBin() {
|
|
|
637
609
|
const wd = detectDirectory(process.argv[1], "package.json");
|
|
638
610
|
return process.argv[1].slice(wd.length + path.sep.length).replace(path.sep, "/");
|
|
639
611
|
}
|
|
612
|
+
// src/date.ts
|
|
613
|
+
var MS_IN_A_DAY = 3600 * 24 * 1000;
|
|
614
|
+
function getDateOnly(date) {
|
|
615
|
+
return new Date(date).toISOString().split("T")[0];
|
|
616
|
+
}
|
|
617
|
+
function findDateAfter(date, n) {
|
|
618
|
+
const d = new Date(date);
|
|
619
|
+
const after = new Date(d.getTime() + MS_IN_A_DAY * n);
|
|
620
|
+
return getDateOnly(after);
|
|
621
|
+
}
|
|
622
|
+
function daysInRange(from, to) {
|
|
623
|
+
const fromTime = new Date(from).getTime();
|
|
624
|
+
const toTime = new Date(to).getTime();
|
|
625
|
+
if (fromTime > toTime) {
|
|
626
|
+
throw new Error(`range to date couldn't be earlier than range from date`);
|
|
627
|
+
}
|
|
628
|
+
const daysBetween = Math.floor((toTime - fromTime) / MS_IN_A_DAY);
|
|
629
|
+
const dates = [getDateOnly(new Date(fromTime))];
|
|
630
|
+
for (let i = 1;i <= daysBetween; i += 1) {
|
|
631
|
+
dates.push(getDateOnly(new Date(fromTime + i * MS_IN_A_DAY)));
|
|
632
|
+
}
|
|
633
|
+
return dates;
|
|
634
|
+
}
|
|
635
|
+
function dateRange(from, to, step) {
|
|
636
|
+
const days = daysInRange(from, to);
|
|
637
|
+
const windows = arrayGroup(days, step);
|
|
638
|
+
return windows.map((w) => [w[0], w[w.length - 1]]);
|
|
639
|
+
}
|
|
640
640
|
// src/indexer.ts
|
|
641
641
|
import meow from "meow";
|
|
642
642
|
var STATE_TABLE_NAME = "skynet-" + getEnvironment() + "-indexer-state";
|
package/examples/api.ts
CHANGED
|
File without changes
|
package/examples/indexer.ts
CHANGED
|
File without changes
|
package/examples/mode-indexer.ts
CHANGED
|
File without changes
|
package/package.json
CHANGED
package/src/deploy.ts
CHANGED
|
@@ -143,7 +143,7 @@ const genConfig = ({
|
|
|
143
143
|
command = "sh"
|
|
144
144
|
args = [
|
|
145
145
|
"-c",
|
|
146
|
-
"cd \${meta.skynet_code_path}/${workingDirectory} && if [ -e bun.lockb ]; then bun install --silent; else yarn install --silent; fi && exec ${cmd}"
|
|
146
|
+
"cd \${meta.skynet_code_path}/${workingDirectory} && if [ -e bun.lockb ] || [ -e bun.lock ]; then bun install --production --silent; else yarn install --production --silent; fi && exec ${cmd}"
|
|
147
147
|
]
|
|
148
148
|
}
|
|
149
149
|
|