@certik/skynet 0.25.2 → 0.25.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/dist/api.js +38 -38
- package/dist/app.js +287 -281
- package/dist/cli.d.ts +2 -1
- package/dist/cli.js +4 -0
- package/dist/deploy.js +22 -19
- package/dist/dynamodb.js +22 -22
- package/dist/indexer.js +134 -131
- package/examples/api.ts +0 -0
- package/examples/indexer.ts +0 -0
- package/examples/mode-indexer.ts +0 -0
- package/package.json +1 -1
- package/src/app.ts +7 -4
- package/src/cli.ts +6 -1
- package/.vscode/settings.json +0 -5
package/CHANGELOG.md
CHANGED
package/dist/api.js
CHANGED
|
@@ -1,41 +1,3 @@
|
|
|
1
|
-
// src/selector.ts
|
|
2
|
-
function getSelectorDesc(selector) {
|
|
3
|
-
return Object.keys(selector).map((name) => {
|
|
4
|
-
return ` --${name.padEnd(14)}${selector[name].desc || selector[name].description || ""}`;
|
|
5
|
-
}).join(`
|
|
6
|
-
`);
|
|
7
|
-
}
|
|
8
|
-
function getSelectorFlags(selector) {
|
|
9
|
-
return Object.keys(selector).reduce((acc, name) => {
|
|
10
|
-
const flag = {
|
|
11
|
-
type: selector[name].type || "string",
|
|
12
|
-
...selector[name]
|
|
13
|
-
};
|
|
14
|
-
if (!selector[name].optional && selector[name].isRequired !== false) {
|
|
15
|
-
flag.isRequired = true;
|
|
16
|
-
}
|
|
17
|
-
return { ...acc, [name]: flag };
|
|
18
|
-
}, {});
|
|
19
|
-
}
|
|
20
|
-
function toSelectorString(selectorFlags, delim = ",") {
|
|
21
|
-
return Object.keys(selectorFlags).sort().map((flag) => {
|
|
22
|
-
return `${flag}=${selectorFlags[flag]}`;
|
|
23
|
-
}).join(delim);
|
|
24
|
-
}
|
|
25
|
-
function normalizeSelectorValue(v) {
|
|
26
|
-
return v.replace(/[^A-Za-z0-9]+/g, "-");
|
|
27
|
-
}
|
|
28
|
-
function getJobName(name, selectorFlags, mode) {
|
|
29
|
-
const selectorNamePart = Object.keys(selectorFlags).sort().map((name2) => selectorFlags[name2]).join("-");
|
|
30
|
-
let jobName = name;
|
|
31
|
-
if (mode) {
|
|
32
|
-
jobName += `-${mode}`;
|
|
33
|
-
}
|
|
34
|
-
if (selectorNamePart.length > 0) {
|
|
35
|
-
jobName += `-${normalizeSelectorValue(selectorNamePart)}`;
|
|
36
|
-
}
|
|
37
|
-
return jobName;
|
|
38
|
-
}
|
|
39
1
|
// src/env.ts
|
|
40
2
|
function ensureAndGet(envName, defaultValue) {
|
|
41
3
|
return process.env[envName] || defaultValue;
|
|
@@ -112,6 +74,44 @@ var logger = {
|
|
|
112
74
|
}
|
|
113
75
|
}
|
|
114
76
|
};
|
|
77
|
+
// src/selector.ts
|
|
78
|
+
function getSelectorDesc(selector) {
|
|
79
|
+
return Object.keys(selector).map((name) => {
|
|
80
|
+
return ` --${name.padEnd(14)}${selector[name].desc || selector[name].description || ""}`;
|
|
81
|
+
}).join(`
|
|
82
|
+
`);
|
|
83
|
+
}
|
|
84
|
+
function getSelectorFlags(selector) {
|
|
85
|
+
return Object.keys(selector).reduce((acc, name) => {
|
|
86
|
+
const flag = {
|
|
87
|
+
type: selector[name].type || "string",
|
|
88
|
+
...selector[name]
|
|
89
|
+
};
|
|
90
|
+
if (!selector[name].optional && selector[name].isRequired !== false) {
|
|
91
|
+
flag.isRequired = true;
|
|
92
|
+
}
|
|
93
|
+
return { ...acc, [name]: flag };
|
|
94
|
+
}, {});
|
|
95
|
+
}
|
|
96
|
+
function toSelectorString(selectorFlags, delim = ",") {
|
|
97
|
+
return Object.keys(selectorFlags).sort().map((flag) => {
|
|
98
|
+
return `${flag}=${selectorFlags[flag]}`;
|
|
99
|
+
}).join(delim);
|
|
100
|
+
}
|
|
101
|
+
function normalizeSelectorValue(v) {
|
|
102
|
+
return v.replace(/[^A-Za-z0-9]+/g, "-");
|
|
103
|
+
}
|
|
104
|
+
function getJobName(name, selectorFlags, mode) {
|
|
105
|
+
const selectorNamePart = Object.keys(selectorFlags).sort().map((name2) => selectorFlags[name2]).join("-");
|
|
106
|
+
let jobName = name;
|
|
107
|
+
if (mode) {
|
|
108
|
+
jobName += `-${mode}`;
|
|
109
|
+
}
|
|
110
|
+
if (selectorNamePart.length > 0) {
|
|
111
|
+
jobName += `-${normalizeSelectorValue(selectorNamePart)}`;
|
|
112
|
+
}
|
|
113
|
+
return jobName;
|
|
114
|
+
}
|
|
115
115
|
// src/api.ts
|
|
116
116
|
import osModule from "os";
|
|
117
117
|
import express from "express";
|
package/dist/app.js
CHANGED
|
@@ -1,41 +1,3 @@
|
|
|
1
|
-
// src/selector.ts
|
|
2
|
-
function getSelectorDesc(selector) {
|
|
3
|
-
return Object.keys(selector).map((name) => {
|
|
4
|
-
return ` --${name.padEnd(14)}${selector[name].desc || selector[name].description || ""}`;
|
|
5
|
-
}).join(`
|
|
6
|
-
`);
|
|
7
|
-
}
|
|
8
|
-
function getSelectorFlags(selector) {
|
|
9
|
-
return Object.keys(selector).reduce((acc, name) => {
|
|
10
|
-
const flag = {
|
|
11
|
-
type: selector[name].type || "string",
|
|
12
|
-
...selector[name]
|
|
13
|
-
};
|
|
14
|
-
if (!selector[name].optional && selector[name].isRequired !== false) {
|
|
15
|
-
flag.isRequired = true;
|
|
16
|
-
}
|
|
17
|
-
return { ...acc, [name]: flag };
|
|
18
|
-
}, {});
|
|
19
|
-
}
|
|
20
|
-
function toSelectorString(selectorFlags, delim = ",") {
|
|
21
|
-
return Object.keys(selectorFlags).sort().map((flag) => {
|
|
22
|
-
return `${flag}=${selectorFlags[flag]}`;
|
|
23
|
-
}).join(delim);
|
|
24
|
-
}
|
|
25
|
-
function normalizeSelectorValue(v) {
|
|
26
|
-
return v.replace(/[^A-Za-z0-9]+/g, "-");
|
|
27
|
-
}
|
|
28
|
-
function getJobName(name, selectorFlags, mode) {
|
|
29
|
-
const selectorNamePart = Object.keys(selectorFlags).sort().map((name2) => selectorFlags[name2]).join("-");
|
|
30
|
-
let jobName = name;
|
|
31
|
-
if (mode) {
|
|
32
|
-
jobName += `-${mode}`;
|
|
33
|
-
}
|
|
34
|
-
if (selectorNamePart.length > 0) {
|
|
35
|
-
jobName += `-${normalizeSelectorValue(selectorNamePart)}`;
|
|
36
|
-
}
|
|
37
|
-
return jobName;
|
|
38
|
-
}
|
|
39
1
|
// src/env.ts
|
|
40
2
|
function ensureAndGet(envName, defaultValue) {
|
|
41
3
|
return process.env[envName] || defaultValue;
|
|
@@ -55,207 +17,56 @@ function isProduction() {
|
|
|
55
17
|
function isDev() {
|
|
56
18
|
return getEnvironment() === "dev";
|
|
57
19
|
}
|
|
58
|
-
// src/
|
|
59
|
-
function
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
if (Array.isArray(o)) {
|
|
64
|
-
return `[${o.map(print).join(", ")}]`;
|
|
65
|
-
}
|
|
66
|
-
if (isObject(o)) {
|
|
67
|
-
return `{${Object.keys(o).map((k) => `${k}: ${o[k]}`).join(", ")}}`;
|
|
20
|
+
// src/util.ts
|
|
21
|
+
function range(startAt, endAt, step) {
|
|
22
|
+
const arr = [];
|
|
23
|
+
for (let i = startAt;i <= endAt; i += step) {
|
|
24
|
+
arr.push([i, Math.min(endAt, i + step - 1)]);
|
|
68
25
|
}
|
|
69
|
-
return
|
|
26
|
+
return arr;
|
|
70
27
|
}
|
|
71
|
-
function
|
|
72
|
-
|
|
73
|
-
for (let i = 0
|
|
74
|
-
|
|
28
|
+
function arrayGroup(array, groupSize) {
|
|
29
|
+
const groups = [];
|
|
30
|
+
for (let i = 0;i < array.length; i += groupSize) {
|
|
31
|
+
groups.push(array.slice(i, i + groupSize));
|
|
75
32
|
}
|
|
76
|
-
return
|
|
77
|
-
}
|
|
78
|
-
function timestamp() {
|
|
79
|
-
return new Date().toISOString();
|
|
33
|
+
return groups;
|
|
80
34
|
}
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
}
|
|
86
|
-
},
|
|
87
|
-
log: function(...args) {
|
|
88
|
-
if (true) {
|
|
89
|
-
console.log(`${timestamp()} ${getLine(args)}`);
|
|
90
|
-
}
|
|
91
|
-
},
|
|
92
|
-
error: function(...args) {
|
|
93
|
-
if (true) {
|
|
94
|
-
console.error(`${timestamp()} ${getLine(args)}`);
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
};
|
|
98
|
-
var logger = {
|
|
99
|
-
debug: function(...args) {
|
|
100
|
-
if (true) {
|
|
101
|
-
console.log(`[${timestamp()}]`, ...args);
|
|
102
|
-
}
|
|
103
|
-
},
|
|
104
|
-
log: function(...args) {
|
|
105
|
-
if (true) {
|
|
106
|
-
console.log(`[${timestamp()}]`, ...args);
|
|
107
|
-
}
|
|
108
|
-
},
|
|
109
|
-
error: function(...args) {
|
|
110
|
-
if (true) {
|
|
111
|
-
console.error(`[${timestamp()}]`, ...args);
|
|
112
|
-
}
|
|
35
|
+
function fillRange(start, end) {
|
|
36
|
+
const result = [];
|
|
37
|
+
for (let i = start;i <= end; i++) {
|
|
38
|
+
result.push(i);
|
|
113
39
|
}
|
|
114
|
-
|
|
115
|
-
// src/api.ts
|
|
116
|
-
import osModule from "os";
|
|
117
|
-
import express from "express";
|
|
118
|
-
import meow from "meow";
|
|
119
|
-
async function logStartMiddleware(_, res, next) {
|
|
120
|
-
const start = new Date;
|
|
121
|
-
res.set("x-requested-at", start.toISOString());
|
|
122
|
-
next();
|
|
40
|
+
return result;
|
|
123
41
|
}
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
42
|
+
// src/date.ts
|
|
43
|
+
var MS_IN_A_DAY = 3600 * 24 * 1000;
|
|
44
|
+
function getDateOnly(date) {
|
|
45
|
+
return new Date(date).toISOString().split("T")[0];
|
|
127
46
|
}
|
|
128
|
-
|
|
129
|
-
const
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
next();
|
|
133
|
-
return;
|
|
134
|
-
}
|
|
135
|
-
const start = new Date(requestedAt).getTime();
|
|
136
|
-
const end = new Date().getTime();
|
|
137
|
-
const logInfo = {
|
|
138
|
-
start,
|
|
139
|
-
end,
|
|
140
|
-
elapsed: `${end - start}ms`,
|
|
141
|
-
endpoint: req.path,
|
|
142
|
-
host: req.hostname,
|
|
143
|
-
status: res.statusCode
|
|
144
|
-
};
|
|
145
|
-
if (res.statusMessage) {
|
|
146
|
-
logInfo.errorMessage = res.statusMessage;
|
|
147
|
-
}
|
|
148
|
-
inline.log(JSON.stringify(logInfo));
|
|
149
|
-
next();
|
|
47
|
+
function findDateAfter(date, n) {
|
|
48
|
+
const d = new Date(date);
|
|
49
|
+
const after = new Date(d.getTime() + MS_IN_A_DAY * n);
|
|
50
|
+
return getDateOnly(after);
|
|
150
51
|
}
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
inline.log("request without api key");
|
|
157
|
-
res.status(400).send("require x-api-key header");
|
|
158
|
-
return;
|
|
159
|
-
}
|
|
160
|
-
if (typeof key === "string") {
|
|
161
|
-
if (apiKey !== key) {
|
|
162
|
-
inline.log("request has an invalid api key");
|
|
163
|
-
res.status(400).send("invalid api key");
|
|
164
|
-
return;
|
|
165
|
-
}
|
|
166
|
-
inline.log(`requested by valid key ${key.slice(0, 6)}`);
|
|
167
|
-
} else {
|
|
168
|
-
const name = key[apiKey];
|
|
169
|
-
if (!name) {
|
|
170
|
-
inline.log("request has an invalid api key");
|
|
171
|
-
res.status(400).send("invalid api key");
|
|
172
|
-
return;
|
|
173
|
-
}
|
|
174
|
-
inline.log(`requested by authorized user ${name}`);
|
|
175
|
-
}
|
|
176
|
-
next();
|
|
177
|
-
} catch (err) {
|
|
178
|
-
inline.log("check api key error", err);
|
|
179
|
-
res.status(500).send("internal error");
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
return requireAPIKey;
|
|
183
|
-
};
|
|
184
|
-
async function startApiApp({
|
|
185
|
-
binaryName,
|
|
186
|
-
name,
|
|
187
|
-
selector = {},
|
|
188
|
-
routes,
|
|
189
|
-
serve,
|
|
190
|
-
beforeListen
|
|
191
|
-
}) {
|
|
192
|
-
const app = express();
|
|
193
|
-
app.use(express.json({ limit: "20mb" }));
|
|
194
|
-
const cli = meow(`
|
|
195
|
-
Usage
|
|
196
|
-
$ ${binaryName} <options>
|
|
197
|
-
|
|
198
|
-
Options
|
|
199
|
-
${getSelectorDesc(selector)}
|
|
200
|
-
--verbose Output debug messages
|
|
201
|
-
`, {
|
|
202
|
-
importMeta: import.meta,
|
|
203
|
-
description: false,
|
|
204
|
-
flags: {
|
|
205
|
-
...getSelectorFlags(selector),
|
|
206
|
-
verbose: {
|
|
207
|
-
type: "boolean",
|
|
208
|
-
default: false
|
|
209
|
-
}
|
|
210
|
-
}
|
|
211
|
-
});
|
|
212
|
-
const { verbose, ...selectorFlags } = cli.flags;
|
|
213
|
-
for (const route of routes) {
|
|
214
|
-
const method = route.method ? route.method.toLowerCase() : "get";
|
|
215
|
-
const middlewares = route.middlewares || [];
|
|
216
|
-
if (route.protected) {
|
|
217
|
-
if (!serve.apiKey) {
|
|
218
|
-
throw new Error("serve.apiKey is required for protected route");
|
|
219
|
-
}
|
|
220
|
-
middlewares.unshift(apiKeyMiddleware(serve.apiKey));
|
|
221
|
-
}
|
|
222
|
-
if (app[method]) {
|
|
223
|
-
if (verbose) {
|
|
224
|
-
inline.log(`registering ${method} ${route.path}`);
|
|
225
|
-
}
|
|
226
|
-
app[method](route.path, contextMiddleware, logStartMiddleware, ...middlewares, async (req, res, next) => {
|
|
227
|
-
try {
|
|
228
|
-
await route.handler({ req, res, ...selectorFlags });
|
|
229
|
-
} catch (routeErr) {
|
|
230
|
-
if (routeErr instanceof Error) {
|
|
231
|
-
inline.log("caught route err", routeErr, routeErr.stack);
|
|
232
|
-
res.status(500).send(`internal server error: ${routeErr.message}`);
|
|
233
|
-
} else {
|
|
234
|
-
inline.log("caught route err", routeErr);
|
|
235
|
-
res.status(500).send("internal server error");
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
|
-
next();
|
|
239
|
-
}, logEndMiddleware);
|
|
240
|
-
}
|
|
241
|
-
}
|
|
242
|
-
if (!routes.some((r) => r.path === "/" && r.method?.toUpperCase() === "GET")) {
|
|
243
|
-
app.get("/", (_, res) => {
|
|
244
|
-
res.send("ok");
|
|
245
|
-
});
|
|
52
|
+
function daysInRange(from, to) {
|
|
53
|
+
const fromTime = new Date(from).getTime();
|
|
54
|
+
const toTime = new Date(to).getTime();
|
|
55
|
+
if (fromTime > toTime) {
|
|
56
|
+
throw new Error(`range to date couldn't be earlier than range from date`);
|
|
246
57
|
}
|
|
247
|
-
|
|
248
|
-
|
|
58
|
+
const daysBetween = Math.floor((toTime - fromTime) / MS_IN_A_DAY);
|
|
59
|
+
const dates = [getDateOnly(new Date(fromTime))];
|
|
60
|
+
for (let i = 1;i <= daysBetween; i += 1) {
|
|
61
|
+
dates.push(getDateOnly(new Date(fromTime + i * MS_IN_A_DAY)));
|
|
249
62
|
}
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
});
|
|
63
|
+
return dates;
|
|
64
|
+
}
|
|
65
|
+
function dateRange(from, to, step) {
|
|
66
|
+
const days = daysInRange(from, to);
|
|
67
|
+
const windows = arrayGroup(days, step);
|
|
68
|
+
return windows.map((w) => [w[0], w[w.length - 1]]);
|
|
257
69
|
}
|
|
258
|
-
|
|
259
70
|
// src/object-hash.ts
|
|
260
71
|
import xh from "@node-rs/xxhash";
|
|
261
72
|
function getHash(obj) {
|
|
@@ -382,28 +193,63 @@ function memoize(func, options) {
|
|
|
382
193
|
}
|
|
383
194
|
return pMemoize(func, options);
|
|
384
195
|
}
|
|
385
|
-
// src/
|
|
386
|
-
function
|
|
387
|
-
|
|
388
|
-
for (let i = startAt;i <= endAt; i += step) {
|
|
389
|
-
arr.push([i, Math.min(endAt, i + step - 1)]);
|
|
390
|
-
}
|
|
391
|
-
return arr;
|
|
196
|
+
// src/log.ts
|
|
197
|
+
function isObject(a) {
|
|
198
|
+
return !!a && a.constructor === Object;
|
|
392
199
|
}
|
|
393
|
-
function
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
groups.push(array.slice(i, i + groupSize));
|
|
200
|
+
function print(o) {
|
|
201
|
+
if (Array.isArray(o)) {
|
|
202
|
+
return `[${o.map(print).join(", ")}]`;
|
|
397
203
|
}
|
|
398
|
-
|
|
204
|
+
if (isObject(o)) {
|
|
205
|
+
return `{${Object.keys(o).map((k) => `${k}: ${o[k]}`).join(", ")}}`;
|
|
206
|
+
}
|
|
207
|
+
return `${o}`;
|
|
399
208
|
}
|
|
400
|
-
function
|
|
401
|
-
|
|
402
|
-
for (let i =
|
|
403
|
-
|
|
209
|
+
function getLine(params) {
|
|
210
|
+
let line = "";
|
|
211
|
+
for (let i = 0, l = params.length;i < l; i++) {
|
|
212
|
+
line += `${print(params[i])} `.replace(/\n/gm, "\t");
|
|
404
213
|
}
|
|
405
|
-
return
|
|
214
|
+
return line.trim();
|
|
215
|
+
}
|
|
216
|
+
function timestamp() {
|
|
217
|
+
return new Date().toISOString();
|
|
406
218
|
}
|
|
219
|
+
var inline = {
|
|
220
|
+
debug: function(...args) {
|
|
221
|
+
if (true) {
|
|
222
|
+
console.log(`${timestamp()} ${getLine(args)}`);
|
|
223
|
+
}
|
|
224
|
+
},
|
|
225
|
+
log: function(...args) {
|
|
226
|
+
if (true) {
|
|
227
|
+
console.log(`${timestamp()} ${getLine(args)}`);
|
|
228
|
+
}
|
|
229
|
+
},
|
|
230
|
+
error: function(...args) {
|
|
231
|
+
if (true) {
|
|
232
|
+
console.error(`${timestamp()} ${getLine(args)}`);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
};
|
|
236
|
+
var logger = {
|
|
237
|
+
debug: function(...args) {
|
|
238
|
+
if (true) {
|
|
239
|
+
console.log(`[${timestamp()}]`, ...args);
|
|
240
|
+
}
|
|
241
|
+
},
|
|
242
|
+
log: function(...args) {
|
|
243
|
+
if (true) {
|
|
244
|
+
console.log(`[${timestamp()}]`, ...args);
|
|
245
|
+
}
|
|
246
|
+
},
|
|
247
|
+
error: function(...args) {
|
|
248
|
+
if (true) {
|
|
249
|
+
console.error(`[${timestamp()}]`, ...args);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
};
|
|
407
253
|
// src/dynamodb.ts
|
|
408
254
|
import {
|
|
409
255
|
DynamoDBDocumentClient,
|
|
@@ -718,6 +564,44 @@ async function deleteRecordsByHashKey(tableName, indexName, hashKeyValue, verbos
|
|
|
718
564
|
}
|
|
719
565
|
return totalDeleted;
|
|
720
566
|
}
|
|
567
|
+
// src/selector.ts
|
|
568
|
+
function getSelectorDesc(selector) {
|
|
569
|
+
return Object.keys(selector).map((name) => {
|
|
570
|
+
return ` --${name.padEnd(14)}${selector[name].desc || selector[name].description || ""}`;
|
|
571
|
+
}).join(`
|
|
572
|
+
`);
|
|
573
|
+
}
|
|
574
|
+
function getSelectorFlags(selector) {
|
|
575
|
+
return Object.keys(selector).reduce((acc, name) => {
|
|
576
|
+
const flag = {
|
|
577
|
+
type: selector[name].type || "string",
|
|
578
|
+
...selector[name]
|
|
579
|
+
};
|
|
580
|
+
if (!selector[name].optional && selector[name].isRequired !== false) {
|
|
581
|
+
flag.isRequired = true;
|
|
582
|
+
}
|
|
583
|
+
return { ...acc, [name]: flag };
|
|
584
|
+
}, {});
|
|
585
|
+
}
|
|
586
|
+
function toSelectorString(selectorFlags, delim = ",") {
|
|
587
|
+
return Object.keys(selectorFlags).sort().map((flag) => {
|
|
588
|
+
return `${flag}=${selectorFlags[flag]}`;
|
|
589
|
+
}).join(delim);
|
|
590
|
+
}
|
|
591
|
+
function normalizeSelectorValue(v) {
|
|
592
|
+
return v.replace(/[^A-Za-z0-9]+/g, "-");
|
|
593
|
+
}
|
|
594
|
+
function getJobName(name, selectorFlags, mode) {
|
|
595
|
+
const selectorNamePart = Object.keys(selectorFlags).sort().map((name2) => selectorFlags[name2]).join("-");
|
|
596
|
+
let jobName = name;
|
|
597
|
+
if (mode) {
|
|
598
|
+
jobName += `-${mode}`;
|
|
599
|
+
}
|
|
600
|
+
if (selectorNamePart.length > 0) {
|
|
601
|
+
jobName += `-${normalizeSelectorValue(selectorNamePart)}`;
|
|
602
|
+
}
|
|
603
|
+
return jobName;
|
|
604
|
+
}
|
|
721
605
|
// src/cli.ts
|
|
722
606
|
import path from "path";
|
|
723
607
|
import fs from "fs";
|
|
@@ -753,36 +637,11 @@ function detectBin() {
|
|
|
753
637
|
const wd = detectDirectory(process.argv[1], "package.json");
|
|
754
638
|
return process.argv[1].slice(wd.length + path.sep.length).replace(path.sep, "/");
|
|
755
639
|
}
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
function getDateOnly(date) {
|
|
759
|
-
return new Date(date).toISOString().split("T")[0];
|
|
760
|
-
}
|
|
761
|
-
function findDateAfter(date, n) {
|
|
762
|
-
const d = new Date(date);
|
|
763
|
-
const after = new Date(d.getTime() + MS_IN_A_DAY * n);
|
|
764
|
-
return getDateOnly(after);
|
|
765
|
-
}
|
|
766
|
-
function daysInRange(from, to) {
|
|
767
|
-
const fromTime = new Date(from).getTime();
|
|
768
|
-
const toTime = new Date(to).getTime();
|
|
769
|
-
if (fromTime > toTime) {
|
|
770
|
-
throw new Error(`range to date couldn't be earlier than range from date`);
|
|
771
|
-
}
|
|
772
|
-
const daysBetween = Math.floor((toTime - fromTime) / MS_IN_A_DAY);
|
|
773
|
-
const dates = [getDateOnly(new Date(fromTime))];
|
|
774
|
-
for (let i = 1;i <= daysBetween; i += 1) {
|
|
775
|
-
dates.push(getDateOnly(new Date(fromTime + i * MS_IN_A_DAY)));
|
|
776
|
-
}
|
|
777
|
-
return dates;
|
|
778
|
-
}
|
|
779
|
-
function dateRange(from, to, step) {
|
|
780
|
-
const days = daysInRange(from, to);
|
|
781
|
-
const windows = arrayGroup(days, step);
|
|
782
|
-
return windows.map((w) => [w[0], w[w.length - 1]]);
|
|
640
|
+
function getDeployBin(bin) {
|
|
641
|
+
return bin.endsWith(".ts") ? `bun ${bin}` : bin;
|
|
783
642
|
}
|
|
784
643
|
// src/indexer.ts
|
|
785
|
-
import
|
|
644
|
+
import meow from "meow";
|
|
786
645
|
var STATE_TABLE_NAME = "skynet-" + getEnvironment() + "-indexer-state";
|
|
787
646
|
async function getIndexerLatestId(name, selectorFlags) {
|
|
788
647
|
const record = await getRecordByKey(STATE_TABLE_NAME, {
|
|
@@ -1125,7 +984,7 @@ function createModeIndexerApp({
|
|
|
1125
984
|
if (!binaryName) {
|
|
1126
985
|
binaryName = getBinaryName();
|
|
1127
986
|
}
|
|
1128
|
-
const cli =
|
|
987
|
+
const cli = meow(`
|
|
1129
988
|
Usage
|
|
1130
989
|
|
|
1131
990
|
$ ${binaryName} <options>
|
|
@@ -1183,7 +1042,7 @@ function createIndexerApp({
|
|
|
1183
1042
|
if (!binaryName) {
|
|
1184
1043
|
binaryName = getBinaryName();
|
|
1185
1044
|
}
|
|
1186
|
-
const cli =
|
|
1045
|
+
const cli = meow(`
|
|
1187
1046
|
Usage
|
|
1188
1047
|
$ ${binaryName} <options>
|
|
1189
1048
|
|
|
@@ -1240,7 +1099,7 @@ ${selector ? getSelectorDesc(selector) : ""}
|
|
|
1240
1099
|
import fs2 from "fs/promises";
|
|
1241
1100
|
import fso from "fs";
|
|
1242
1101
|
import { execa } from "execa";
|
|
1243
|
-
import
|
|
1102
|
+
import meow2 from "meow";
|
|
1244
1103
|
import chalk from "chalk";
|
|
1245
1104
|
import which from "which";
|
|
1246
1105
|
var INTERVAL_ALIASES = {
|
|
@@ -1547,7 +1406,7 @@ function createModeDeploy({
|
|
|
1547
1406
|
if (!binaryName) {
|
|
1548
1407
|
binaryName = getBinaryName();
|
|
1549
1408
|
}
|
|
1550
|
-
const cli =
|
|
1409
|
+
const cli = meow2(`
|
|
1551
1410
|
Usage
|
|
1552
1411
|
|
|
1553
1412
|
$ ${binaryName} <options>
|
|
@@ -1677,7 +1536,7 @@ function createDeploy({
|
|
|
1677
1536
|
if (!binaryName) {
|
|
1678
1537
|
binaryName = getBinaryName();
|
|
1679
1538
|
}
|
|
1680
|
-
const cli =
|
|
1539
|
+
const cli = meow2(`
|
|
1681
1540
|
Usage
|
|
1682
1541
|
|
|
1683
1542
|
$ ${binaryName} <options>
|
|
@@ -1725,6 +1584,150 @@ ${getSelectorDesc(selector)}
|
|
|
1725
1584
|
}
|
|
1726
1585
|
return { deploy };
|
|
1727
1586
|
}
|
|
1587
|
+
// src/api.ts
|
|
1588
|
+
import osModule from "os";
|
|
1589
|
+
import express from "express";
|
|
1590
|
+
import meow3 from "meow";
|
|
1591
|
+
async function logStartMiddleware(_, res, next) {
|
|
1592
|
+
const start = new Date;
|
|
1593
|
+
res.set("x-requested-at", start.toISOString());
|
|
1594
|
+
next();
|
|
1595
|
+
}
|
|
1596
|
+
async function contextMiddleware(_, res, next) {
|
|
1597
|
+
res.set("x-instance-id", osModule.hostname());
|
|
1598
|
+
next();
|
|
1599
|
+
}
|
|
1600
|
+
async function logEndMiddleware(req, res, next) {
|
|
1601
|
+
const requestedAt = res.get("x-requested-at");
|
|
1602
|
+
if (!requestedAt) {
|
|
1603
|
+
inline.log("missing x-requested-at header");
|
|
1604
|
+
next();
|
|
1605
|
+
return;
|
|
1606
|
+
}
|
|
1607
|
+
const start = new Date(requestedAt).getTime();
|
|
1608
|
+
const end = new Date().getTime();
|
|
1609
|
+
const logInfo = {
|
|
1610
|
+
start,
|
|
1611
|
+
end,
|
|
1612
|
+
elapsed: `${end - start}ms`,
|
|
1613
|
+
endpoint: req.path,
|
|
1614
|
+
host: req.hostname,
|
|
1615
|
+
status: res.statusCode
|
|
1616
|
+
};
|
|
1617
|
+
if (res.statusMessage) {
|
|
1618
|
+
logInfo.errorMessage = res.statusMessage;
|
|
1619
|
+
}
|
|
1620
|
+
inline.log(JSON.stringify(logInfo));
|
|
1621
|
+
next();
|
|
1622
|
+
}
|
|
1623
|
+
var apiKeyMiddleware = (key) => {
|
|
1624
|
+
async function requireAPIKey(req, res, next) {
|
|
1625
|
+
try {
|
|
1626
|
+
const apiKey = req.get("x-api-key") || req.query["api-key"];
|
|
1627
|
+
if (!apiKey) {
|
|
1628
|
+
inline.log("request without api key");
|
|
1629
|
+
res.status(400).send("require x-api-key header");
|
|
1630
|
+
return;
|
|
1631
|
+
}
|
|
1632
|
+
if (typeof key === "string") {
|
|
1633
|
+
if (apiKey !== key) {
|
|
1634
|
+
inline.log("request has an invalid api key");
|
|
1635
|
+
res.status(400).send("invalid api key");
|
|
1636
|
+
return;
|
|
1637
|
+
}
|
|
1638
|
+
inline.log(`requested by valid key ${key.slice(0, 6)}`);
|
|
1639
|
+
} else {
|
|
1640
|
+
const name = key[apiKey];
|
|
1641
|
+
if (!name) {
|
|
1642
|
+
inline.log("request has an invalid api key");
|
|
1643
|
+
res.status(400).send("invalid api key");
|
|
1644
|
+
return;
|
|
1645
|
+
}
|
|
1646
|
+
inline.log(`requested by authorized user ${name}`);
|
|
1647
|
+
}
|
|
1648
|
+
next();
|
|
1649
|
+
} catch (err) {
|
|
1650
|
+
inline.log("check api key error", err);
|
|
1651
|
+
res.status(500).send("internal error");
|
|
1652
|
+
}
|
|
1653
|
+
}
|
|
1654
|
+
return requireAPIKey;
|
|
1655
|
+
};
|
|
1656
|
+
async function startApiApp({
|
|
1657
|
+
binaryName,
|
|
1658
|
+
name,
|
|
1659
|
+
selector = {},
|
|
1660
|
+
routes,
|
|
1661
|
+
serve,
|
|
1662
|
+
beforeListen
|
|
1663
|
+
}) {
|
|
1664
|
+
const app = express();
|
|
1665
|
+
app.use(express.json({ limit: "20mb" }));
|
|
1666
|
+
const cli = meow3(`
|
|
1667
|
+
Usage
|
|
1668
|
+
$ ${binaryName} <options>
|
|
1669
|
+
|
|
1670
|
+
Options
|
|
1671
|
+
${getSelectorDesc(selector)}
|
|
1672
|
+
--verbose Output debug messages
|
|
1673
|
+
`, {
|
|
1674
|
+
importMeta: import.meta,
|
|
1675
|
+
description: false,
|
|
1676
|
+
flags: {
|
|
1677
|
+
...getSelectorFlags(selector),
|
|
1678
|
+
verbose: {
|
|
1679
|
+
type: "boolean",
|
|
1680
|
+
default: false
|
|
1681
|
+
}
|
|
1682
|
+
}
|
|
1683
|
+
});
|
|
1684
|
+
const { verbose, ...selectorFlags } = cli.flags;
|
|
1685
|
+
for (const route of routes) {
|
|
1686
|
+
const method = route.method ? route.method.toLowerCase() : "get";
|
|
1687
|
+
const middlewares = route.middlewares || [];
|
|
1688
|
+
if (route.protected) {
|
|
1689
|
+
if (!serve.apiKey) {
|
|
1690
|
+
throw new Error("serve.apiKey is required for protected route");
|
|
1691
|
+
}
|
|
1692
|
+
middlewares.unshift(apiKeyMiddleware(serve.apiKey));
|
|
1693
|
+
}
|
|
1694
|
+
if (app[method]) {
|
|
1695
|
+
if (verbose) {
|
|
1696
|
+
inline.log(`registering ${method} ${route.path}`);
|
|
1697
|
+
}
|
|
1698
|
+
app[method](route.path, contextMiddleware, logStartMiddleware, ...middlewares, async (req, res, next) => {
|
|
1699
|
+
try {
|
|
1700
|
+
await route.handler({ req, res, ...selectorFlags });
|
|
1701
|
+
} catch (routeErr) {
|
|
1702
|
+
if (routeErr instanceof Error) {
|
|
1703
|
+
inline.log("caught route err", routeErr, routeErr.stack);
|
|
1704
|
+
res.status(500).send(`internal server error: ${routeErr.message}`);
|
|
1705
|
+
} else {
|
|
1706
|
+
inline.log("caught route err", routeErr);
|
|
1707
|
+
res.status(500).send("internal server error");
|
|
1708
|
+
}
|
|
1709
|
+
}
|
|
1710
|
+
next();
|
|
1711
|
+
}, logEndMiddleware);
|
|
1712
|
+
}
|
|
1713
|
+
}
|
|
1714
|
+
if (!routes.some((r) => r.path === "/" && r.method?.toUpperCase() === "GET")) {
|
|
1715
|
+
app.get("/", (_, res) => {
|
|
1716
|
+
res.send("ok");
|
|
1717
|
+
});
|
|
1718
|
+
}
|
|
1719
|
+
if (beforeListen) {
|
|
1720
|
+
await beforeListen({ app });
|
|
1721
|
+
}
|
|
1722
|
+
app.listen(serve.port, () => {
|
|
1723
|
+
if (isProduction()) {
|
|
1724
|
+
inline.log(`${name} listening at https://api.wf.corp.certik.com${serve.prefix}`);
|
|
1725
|
+
} else {
|
|
1726
|
+
inline.log(`${name} listening at http://localhost:${serve.port}`);
|
|
1727
|
+
}
|
|
1728
|
+
});
|
|
1729
|
+
}
|
|
1730
|
+
|
|
1728
1731
|
// src/app.ts
|
|
1729
1732
|
import { EOL } from "os";
|
|
1730
1733
|
function printAppHelp() {
|
|
@@ -1834,12 +1837,13 @@ function indexer({
|
|
|
1834
1837
|
},
|
|
1835
1838
|
onDeploy: () => {
|
|
1836
1839
|
const bin = detectBin();
|
|
1840
|
+
const runBin = getDeployBin(bin);
|
|
1837
1841
|
const needDoppler = Object.values(env).some((v) => v === SENSITIVE_VALUE);
|
|
1838
1842
|
const { deploy } = createDeploy({
|
|
1839
1843
|
binaryName: `${getBinaryName()} deploy`,
|
|
1840
1844
|
name,
|
|
1841
1845
|
workingDirectory: detectWorkingDirectory(),
|
|
1842
|
-
bin: needDoppler ? `doppler run -- ${
|
|
1846
|
+
bin: needDoppler ? `doppler run -- ${runBin} run` : `${runBin} run`,
|
|
1843
1847
|
selector,
|
|
1844
1848
|
region,
|
|
1845
1849
|
env,
|
|
@@ -1917,12 +1921,13 @@ function modeIndexer({
|
|
|
1917
1921
|
},
|
|
1918
1922
|
onDeploy: () => {
|
|
1919
1923
|
const bin = detectBin();
|
|
1924
|
+
const runBin = getDeployBin(bin);
|
|
1920
1925
|
const needDoppler = Object.values(env).some((v) => v === SENSITIVE_VALUE);
|
|
1921
1926
|
const { deploy } = createModeDeploy({
|
|
1922
1927
|
binaryName: `${getBinaryName()} deploy`,
|
|
1923
1928
|
name,
|
|
1924
1929
|
workingDirectory: detectWorkingDirectory(),
|
|
1925
|
-
bin: needDoppler ? `doppler run -- ${
|
|
1930
|
+
bin: needDoppler ? `doppler run -- ${runBin} run` : `${runBin} run`,
|
|
1926
1931
|
selector,
|
|
1927
1932
|
region,
|
|
1928
1933
|
env,
|
|
@@ -2012,12 +2017,13 @@ function api({
|
|
|
2012
2017
|
},
|
|
2013
2018
|
onDeploy: () => {
|
|
2014
2019
|
const bin = detectBin();
|
|
2020
|
+
const runBin = getDeployBin(bin);
|
|
2015
2021
|
const needDoppler = Object.values(env).some((v) => v === SENSITIVE_VALUE);
|
|
2016
2022
|
const { deploy } = createDeploy({
|
|
2017
2023
|
binaryName: `${getBinaryName()} deploy`,
|
|
2018
2024
|
name,
|
|
2019
2025
|
workingDirectory: detectWorkingDirectory(),
|
|
2020
|
-
bin: needDoppler ? `doppler run -- ${
|
|
2026
|
+
bin: needDoppler ? `doppler run -- ${runBin} run` : `${runBin} run`,
|
|
2021
2027
|
selector,
|
|
2022
2028
|
region,
|
|
2023
2029
|
env,
|
package/dist/cli.d.ts
CHANGED
|
@@ -2,4 +2,5 @@ declare function getBinaryName(): string;
|
|
|
2
2
|
declare function detectSkynetDirectory(): string;
|
|
3
3
|
declare function detectWorkingDirectory(): string;
|
|
4
4
|
declare function detectBin(): string;
|
|
5
|
-
|
|
5
|
+
declare function getDeployBin(bin: string): string;
|
|
6
|
+
export { getBinaryName, detectSkynetDirectory, detectWorkingDirectory, detectBin, getDeployBin };
|
package/dist/cli.js
CHANGED
|
@@ -33,7 +33,11 @@ function detectBin() {
|
|
|
33
33
|
const wd = detectDirectory(process.argv[1], "package.json");
|
|
34
34
|
return process.argv[1].slice(wd.length + path.sep.length).replace(path.sep, "/");
|
|
35
35
|
}
|
|
36
|
+
function getDeployBin(bin) {
|
|
37
|
+
return bin.endsWith(".ts") ? `bun ${bin}` : bin;
|
|
38
|
+
}
|
|
36
39
|
export {
|
|
40
|
+
getDeployBin,
|
|
37
41
|
getBinaryName,
|
|
38
42
|
detectWorkingDirectory,
|
|
39
43
|
detectSkynetDirectory,
|
package/dist/deploy.js
CHANGED
|
@@ -1,3 +1,22 @@
|
|
|
1
|
+
// src/env.ts
|
|
2
|
+
function ensureAndGet(envName, defaultValue) {
|
|
3
|
+
return process.env[envName] || defaultValue;
|
|
4
|
+
}
|
|
5
|
+
function getEnvironment() {
|
|
6
|
+
return ensureAndGet("SKYNET_ENVIRONMENT", "dev");
|
|
7
|
+
}
|
|
8
|
+
function getEnvOrThrow(envName) {
|
|
9
|
+
if (!process.env[envName]) {
|
|
10
|
+
throw new Error(`Must set environment variable ${envName}`);
|
|
11
|
+
}
|
|
12
|
+
return process.env[envName];
|
|
13
|
+
}
|
|
14
|
+
function isProduction() {
|
|
15
|
+
return getEnvironment() === "prd";
|
|
16
|
+
}
|
|
17
|
+
function isDev() {
|
|
18
|
+
return getEnvironment() === "dev";
|
|
19
|
+
}
|
|
1
20
|
// src/selector.ts
|
|
2
21
|
function getSelectorDesc(selector) {
|
|
3
22
|
return Object.keys(selector).map((name) => {
|
|
@@ -36,25 +55,6 @@ function getJobName(name, selectorFlags, mode) {
|
|
|
36
55
|
}
|
|
37
56
|
return jobName;
|
|
38
57
|
}
|
|
39
|
-
// src/env.ts
|
|
40
|
-
function ensureAndGet(envName, defaultValue) {
|
|
41
|
-
return process.env[envName] || defaultValue;
|
|
42
|
-
}
|
|
43
|
-
function getEnvironment() {
|
|
44
|
-
return ensureAndGet("SKYNET_ENVIRONMENT", "dev");
|
|
45
|
-
}
|
|
46
|
-
function getEnvOrThrow(envName) {
|
|
47
|
-
if (!process.env[envName]) {
|
|
48
|
-
throw new Error(`Must set environment variable ${envName}`);
|
|
49
|
-
}
|
|
50
|
-
return process.env[envName];
|
|
51
|
-
}
|
|
52
|
-
function isProduction() {
|
|
53
|
-
return getEnvironment() === "prd";
|
|
54
|
-
}
|
|
55
|
-
function isDev() {
|
|
56
|
-
return getEnvironment() === "dev";
|
|
57
|
-
}
|
|
58
58
|
// src/cli.ts
|
|
59
59
|
import path from "path";
|
|
60
60
|
import fs from "fs";
|
|
@@ -90,6 +90,9 @@ function detectBin() {
|
|
|
90
90
|
const wd = detectDirectory(process.argv[1], "package.json");
|
|
91
91
|
return process.argv[1].slice(wd.length + path.sep.length).replace(path.sep, "/");
|
|
92
92
|
}
|
|
93
|
+
function getDeployBin(bin) {
|
|
94
|
+
return bin.endsWith(".ts") ? `bun ${bin}` : bin;
|
|
95
|
+
}
|
|
93
96
|
// src/deploy.ts
|
|
94
97
|
import fs2 from "fs/promises";
|
|
95
98
|
import fso from "fs";
|
package/dist/dynamodb.js
CHANGED
|
@@ -1,3 +1,25 @@
|
|
|
1
|
+
// src/util.ts
|
|
2
|
+
function range(startAt, endAt, step) {
|
|
3
|
+
const arr = [];
|
|
4
|
+
for (let i = startAt;i <= endAt; i += step) {
|
|
5
|
+
arr.push([i, Math.min(endAt, i + step - 1)]);
|
|
6
|
+
}
|
|
7
|
+
return arr;
|
|
8
|
+
}
|
|
9
|
+
function arrayGroup(array, groupSize) {
|
|
10
|
+
const groups = [];
|
|
11
|
+
for (let i = 0;i < array.length; i += groupSize) {
|
|
12
|
+
groups.push(array.slice(i, i + groupSize));
|
|
13
|
+
}
|
|
14
|
+
return groups;
|
|
15
|
+
}
|
|
16
|
+
function fillRange(start, end) {
|
|
17
|
+
const result = [];
|
|
18
|
+
for (let i = start;i <= end; i++) {
|
|
19
|
+
result.push(i);
|
|
20
|
+
}
|
|
21
|
+
return result;
|
|
22
|
+
}
|
|
1
23
|
// src/object-hash.ts
|
|
2
24
|
import xh from "@node-rs/xxhash";
|
|
3
25
|
function getHash(obj) {
|
|
@@ -124,28 +146,6 @@ function memoize(func, options) {
|
|
|
124
146
|
}
|
|
125
147
|
return pMemoize(func, options);
|
|
126
148
|
}
|
|
127
|
-
// src/util.ts
|
|
128
|
-
function range(startAt, endAt, step) {
|
|
129
|
-
const arr = [];
|
|
130
|
-
for (let i = startAt;i <= endAt; i += step) {
|
|
131
|
-
arr.push([i, Math.min(endAt, i + step - 1)]);
|
|
132
|
-
}
|
|
133
|
-
return arr;
|
|
134
|
-
}
|
|
135
|
-
function arrayGroup(array, groupSize) {
|
|
136
|
-
const groups = [];
|
|
137
|
-
for (let i = 0;i < array.length; i += groupSize) {
|
|
138
|
-
groups.push(array.slice(i, i + groupSize));
|
|
139
|
-
}
|
|
140
|
-
return groups;
|
|
141
|
-
}
|
|
142
|
-
function fillRange(start, end) {
|
|
143
|
-
const result = [];
|
|
144
|
-
for (let i = start;i <= end; i++) {
|
|
145
|
-
result.push(i);
|
|
146
|
-
}
|
|
147
|
-
return result;
|
|
148
|
-
}
|
|
149
149
|
// src/dynamodb.ts
|
|
150
150
|
import {
|
|
151
151
|
DynamoDBDocumentClient,
|
package/dist/indexer.js
CHANGED
|
@@ -1,41 +1,3 @@
|
|
|
1
|
-
// src/selector.ts
|
|
2
|
-
function getSelectorDesc(selector) {
|
|
3
|
-
return Object.keys(selector).map((name) => {
|
|
4
|
-
return ` --${name.padEnd(14)}${selector[name].desc || selector[name].description || ""}`;
|
|
5
|
-
}).join(`
|
|
6
|
-
`);
|
|
7
|
-
}
|
|
8
|
-
function getSelectorFlags(selector) {
|
|
9
|
-
return Object.keys(selector).reduce((acc, name) => {
|
|
10
|
-
const flag = {
|
|
11
|
-
type: selector[name].type || "string",
|
|
12
|
-
...selector[name]
|
|
13
|
-
};
|
|
14
|
-
if (!selector[name].optional && selector[name].isRequired !== false) {
|
|
15
|
-
flag.isRequired = true;
|
|
16
|
-
}
|
|
17
|
-
return { ...acc, [name]: flag };
|
|
18
|
-
}, {});
|
|
19
|
-
}
|
|
20
|
-
function toSelectorString(selectorFlags, delim = ",") {
|
|
21
|
-
return Object.keys(selectorFlags).sort().map((flag) => {
|
|
22
|
-
return `${flag}=${selectorFlags[flag]}`;
|
|
23
|
-
}).join(delim);
|
|
24
|
-
}
|
|
25
|
-
function normalizeSelectorValue(v) {
|
|
26
|
-
return v.replace(/[^A-Za-z0-9]+/g, "-");
|
|
27
|
-
}
|
|
28
|
-
function getJobName(name, selectorFlags, mode) {
|
|
29
|
-
const selectorNamePart = Object.keys(selectorFlags).sort().map((name2) => selectorFlags[name2]).join("-");
|
|
30
|
-
let jobName = name;
|
|
31
|
-
if (mode) {
|
|
32
|
-
jobName += `-${mode}`;
|
|
33
|
-
}
|
|
34
|
-
if (selectorNamePart.length > 0) {
|
|
35
|
-
jobName += `-${normalizeSelectorValue(selectorNamePart)}`;
|
|
36
|
-
}
|
|
37
|
-
return jobName;
|
|
38
|
-
}
|
|
39
1
|
// src/env.ts
|
|
40
2
|
function ensureAndGet(envName, defaultValue) {
|
|
41
3
|
return process.env[envName] || defaultValue;
|
|
@@ -55,63 +17,56 @@ function isProduction() {
|
|
|
55
17
|
function isDev() {
|
|
56
18
|
return getEnvironment() === "dev";
|
|
57
19
|
}
|
|
58
|
-
// src/
|
|
59
|
-
function
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
if (Array.isArray(o)) {
|
|
64
|
-
return `[${o.map(print).join(", ")}]`;
|
|
20
|
+
// src/util.ts
|
|
21
|
+
function range(startAt, endAt, step) {
|
|
22
|
+
const arr = [];
|
|
23
|
+
for (let i = startAt;i <= endAt; i += step) {
|
|
24
|
+
arr.push([i, Math.min(endAt, i + step - 1)]);
|
|
65
25
|
}
|
|
66
|
-
|
|
67
|
-
|
|
26
|
+
return arr;
|
|
27
|
+
}
|
|
28
|
+
function arrayGroup(array, groupSize) {
|
|
29
|
+
const groups = [];
|
|
30
|
+
for (let i = 0;i < array.length; i += groupSize) {
|
|
31
|
+
groups.push(array.slice(i, i + groupSize));
|
|
68
32
|
}
|
|
69
|
-
return
|
|
33
|
+
return groups;
|
|
70
34
|
}
|
|
71
|
-
function
|
|
72
|
-
|
|
73
|
-
for (let i =
|
|
74
|
-
|
|
35
|
+
function fillRange(start, end) {
|
|
36
|
+
const result = [];
|
|
37
|
+
for (let i = start;i <= end; i++) {
|
|
38
|
+
result.push(i);
|
|
75
39
|
}
|
|
76
|
-
return
|
|
40
|
+
return result;
|
|
77
41
|
}
|
|
78
|
-
|
|
79
|
-
|
|
42
|
+
// src/date.ts
|
|
43
|
+
var MS_IN_A_DAY = 3600 * 24 * 1000;
|
|
44
|
+
function getDateOnly(date) {
|
|
45
|
+
return new Date(date).toISOString().split("T")[0];
|
|
80
46
|
}
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
},
|
|
92
|
-
error: function(...args) {
|
|
93
|
-
if (true) {
|
|
94
|
-
console.error(`${timestamp()} ${getLine(args)}`);
|
|
95
|
-
}
|
|
47
|
+
function findDateAfter(date, n) {
|
|
48
|
+
const d = new Date(date);
|
|
49
|
+
const after = new Date(d.getTime() + MS_IN_A_DAY * n);
|
|
50
|
+
return getDateOnly(after);
|
|
51
|
+
}
|
|
52
|
+
function daysInRange(from, to) {
|
|
53
|
+
const fromTime = new Date(from).getTime();
|
|
54
|
+
const toTime = new Date(to).getTime();
|
|
55
|
+
if (fromTime > toTime) {
|
|
56
|
+
throw new Error(`range to date couldn't be earlier than range from date`);
|
|
96
57
|
}
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
console.log(`[${timestamp()}]`, ...args);
|
|
102
|
-
}
|
|
103
|
-
},
|
|
104
|
-
log: function(...args) {
|
|
105
|
-
if (true) {
|
|
106
|
-
console.log(`[${timestamp()}]`, ...args);
|
|
107
|
-
}
|
|
108
|
-
},
|
|
109
|
-
error: function(...args) {
|
|
110
|
-
if (true) {
|
|
111
|
-
console.error(`[${timestamp()}]`, ...args);
|
|
112
|
-
}
|
|
58
|
+
const daysBetween = Math.floor((toTime - fromTime) / MS_IN_A_DAY);
|
|
59
|
+
const dates = [getDateOnly(new Date(fromTime))];
|
|
60
|
+
for (let i = 1;i <= daysBetween; i += 1) {
|
|
61
|
+
dates.push(getDateOnly(new Date(fromTime + i * MS_IN_A_DAY)));
|
|
113
62
|
}
|
|
114
|
-
|
|
63
|
+
return dates;
|
|
64
|
+
}
|
|
65
|
+
function dateRange(from, to, step) {
|
|
66
|
+
const days = daysInRange(from, to);
|
|
67
|
+
const windows = arrayGroup(days, step);
|
|
68
|
+
return windows.map((w) => [w[0], w[w.length - 1]]);
|
|
69
|
+
}
|
|
115
70
|
// src/object-hash.ts
|
|
116
71
|
import xh from "@node-rs/xxhash";
|
|
117
72
|
function getHash(obj) {
|
|
@@ -238,28 +193,63 @@ function memoize(func, options) {
|
|
|
238
193
|
}
|
|
239
194
|
return pMemoize(func, options);
|
|
240
195
|
}
|
|
241
|
-
// src/
|
|
242
|
-
function
|
|
243
|
-
|
|
244
|
-
for (let i = startAt;i <= endAt; i += step) {
|
|
245
|
-
arr.push([i, Math.min(endAt, i + step - 1)]);
|
|
246
|
-
}
|
|
247
|
-
return arr;
|
|
196
|
+
// src/log.ts
|
|
197
|
+
function isObject(a) {
|
|
198
|
+
return !!a && a.constructor === Object;
|
|
248
199
|
}
|
|
249
|
-
function
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
groups.push(array.slice(i, i + groupSize));
|
|
200
|
+
function print(o) {
|
|
201
|
+
if (Array.isArray(o)) {
|
|
202
|
+
return `[${o.map(print).join(", ")}]`;
|
|
253
203
|
}
|
|
254
|
-
|
|
204
|
+
if (isObject(o)) {
|
|
205
|
+
return `{${Object.keys(o).map((k) => `${k}: ${o[k]}`).join(", ")}}`;
|
|
206
|
+
}
|
|
207
|
+
return `${o}`;
|
|
255
208
|
}
|
|
256
|
-
function
|
|
257
|
-
|
|
258
|
-
for (let i =
|
|
259
|
-
|
|
209
|
+
function getLine(params) {
|
|
210
|
+
let line = "";
|
|
211
|
+
for (let i = 0, l = params.length;i < l; i++) {
|
|
212
|
+
line += `${print(params[i])} `.replace(/\n/gm, "\t");
|
|
260
213
|
}
|
|
261
|
-
return
|
|
214
|
+
return line.trim();
|
|
262
215
|
}
|
|
216
|
+
function timestamp() {
|
|
217
|
+
return new Date().toISOString();
|
|
218
|
+
}
|
|
219
|
+
var inline = {
|
|
220
|
+
debug: function(...args) {
|
|
221
|
+
if (true) {
|
|
222
|
+
console.log(`${timestamp()} ${getLine(args)}`);
|
|
223
|
+
}
|
|
224
|
+
},
|
|
225
|
+
log: function(...args) {
|
|
226
|
+
if (true) {
|
|
227
|
+
console.log(`${timestamp()} ${getLine(args)}`);
|
|
228
|
+
}
|
|
229
|
+
},
|
|
230
|
+
error: function(...args) {
|
|
231
|
+
if (true) {
|
|
232
|
+
console.error(`${timestamp()} ${getLine(args)}`);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
};
|
|
236
|
+
var logger = {
|
|
237
|
+
debug: function(...args) {
|
|
238
|
+
if (true) {
|
|
239
|
+
console.log(`[${timestamp()}]`, ...args);
|
|
240
|
+
}
|
|
241
|
+
},
|
|
242
|
+
log: function(...args) {
|
|
243
|
+
if (true) {
|
|
244
|
+
console.log(`[${timestamp()}]`, ...args);
|
|
245
|
+
}
|
|
246
|
+
},
|
|
247
|
+
error: function(...args) {
|
|
248
|
+
if (true) {
|
|
249
|
+
console.error(`[${timestamp()}]`, ...args);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
};
|
|
263
253
|
// src/dynamodb.ts
|
|
264
254
|
import {
|
|
265
255
|
DynamoDBDocumentClient,
|
|
@@ -574,6 +564,44 @@ async function deleteRecordsByHashKey(tableName, indexName, hashKeyValue, verbos
|
|
|
574
564
|
}
|
|
575
565
|
return totalDeleted;
|
|
576
566
|
}
|
|
567
|
+
// src/selector.ts
|
|
568
|
+
function getSelectorDesc(selector) {
|
|
569
|
+
return Object.keys(selector).map((name) => {
|
|
570
|
+
return ` --${name.padEnd(14)}${selector[name].desc || selector[name].description || ""}`;
|
|
571
|
+
}).join(`
|
|
572
|
+
`);
|
|
573
|
+
}
|
|
574
|
+
function getSelectorFlags(selector) {
|
|
575
|
+
return Object.keys(selector).reduce((acc, name) => {
|
|
576
|
+
const flag = {
|
|
577
|
+
type: selector[name].type || "string",
|
|
578
|
+
...selector[name]
|
|
579
|
+
};
|
|
580
|
+
if (!selector[name].optional && selector[name].isRequired !== false) {
|
|
581
|
+
flag.isRequired = true;
|
|
582
|
+
}
|
|
583
|
+
return { ...acc, [name]: flag };
|
|
584
|
+
}, {});
|
|
585
|
+
}
|
|
586
|
+
function toSelectorString(selectorFlags, delim = ",") {
|
|
587
|
+
return Object.keys(selectorFlags).sort().map((flag) => {
|
|
588
|
+
return `${flag}=${selectorFlags[flag]}`;
|
|
589
|
+
}).join(delim);
|
|
590
|
+
}
|
|
591
|
+
function normalizeSelectorValue(v) {
|
|
592
|
+
return v.replace(/[^A-Za-z0-9]+/g, "-");
|
|
593
|
+
}
|
|
594
|
+
function getJobName(name, selectorFlags, mode) {
|
|
595
|
+
const selectorNamePart = Object.keys(selectorFlags).sort().map((name2) => selectorFlags[name2]).join("-");
|
|
596
|
+
let jobName = name;
|
|
597
|
+
if (mode) {
|
|
598
|
+
jobName += `-${mode}`;
|
|
599
|
+
}
|
|
600
|
+
if (selectorNamePart.length > 0) {
|
|
601
|
+
jobName += `-${normalizeSelectorValue(selectorNamePart)}`;
|
|
602
|
+
}
|
|
603
|
+
return jobName;
|
|
604
|
+
}
|
|
577
605
|
// src/cli.ts
|
|
578
606
|
import path from "path";
|
|
579
607
|
import fs from "fs";
|
|
@@ -609,33 +637,8 @@ function detectBin() {
|
|
|
609
637
|
const wd = detectDirectory(process.argv[1], "package.json");
|
|
610
638
|
return process.argv[1].slice(wd.length + path.sep.length).replace(path.sep, "/");
|
|
611
639
|
}
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
function getDateOnly(date) {
|
|
615
|
-
return new Date(date).toISOString().split("T")[0];
|
|
616
|
-
}
|
|
617
|
-
function findDateAfter(date, n) {
|
|
618
|
-
const d = new Date(date);
|
|
619
|
-
const after = new Date(d.getTime() + MS_IN_A_DAY * n);
|
|
620
|
-
return getDateOnly(after);
|
|
621
|
-
}
|
|
622
|
-
function daysInRange(from, to) {
|
|
623
|
-
const fromTime = new Date(from).getTime();
|
|
624
|
-
const toTime = new Date(to).getTime();
|
|
625
|
-
if (fromTime > toTime) {
|
|
626
|
-
throw new Error(`range to date couldn't be earlier than range from date`);
|
|
627
|
-
}
|
|
628
|
-
const daysBetween = Math.floor((toTime - fromTime) / MS_IN_A_DAY);
|
|
629
|
-
const dates = [getDateOnly(new Date(fromTime))];
|
|
630
|
-
for (let i = 1;i <= daysBetween; i += 1) {
|
|
631
|
-
dates.push(getDateOnly(new Date(fromTime + i * MS_IN_A_DAY)));
|
|
632
|
-
}
|
|
633
|
-
return dates;
|
|
634
|
-
}
|
|
635
|
-
function dateRange(from, to, step) {
|
|
636
|
-
const days = daysInRange(from, to);
|
|
637
|
-
const windows = arrayGroup(days, step);
|
|
638
|
-
return windows.map((w) => [w[0], w[w.length - 1]]);
|
|
640
|
+
function getDeployBin(bin) {
|
|
641
|
+
return bin.endsWith(".ts") ? `bun ${bin}` : bin;
|
|
639
642
|
}
|
|
640
643
|
// src/indexer.ts
|
|
641
644
|
import meow from "meow";
|
package/examples/api.ts
CHANGED
|
File without changes
|
package/examples/indexer.ts
CHANGED
|
File without changes
|
package/examples/mode-indexer.ts
CHANGED
|
File without changes
|
package/package.json
CHANGED
package/src/app.ts
CHANGED
|
@@ -5,7 +5,7 @@ import { createDeploy, createModeDeploy } from "./deploy";
|
|
|
5
5
|
import type { Env } from "./deploy";
|
|
6
6
|
import { startApiApp } from "./api.js";
|
|
7
7
|
import type { Middleware } from "./api";
|
|
8
|
-
import { getBinaryName, detectBin, detectWorkingDirectory } from "./cli.js";
|
|
8
|
+
import { getBinaryName, detectBin, detectWorkingDirectory, getDeployBin } from "./cli.js";
|
|
9
9
|
import type { Selector } from "./selector";
|
|
10
10
|
import type { Request, Response, Application } from "express";
|
|
11
11
|
|
|
@@ -203,13 +203,14 @@ function indexer<TSelector extends Selector>({
|
|
|
203
203
|
},
|
|
204
204
|
onDeploy: () => {
|
|
205
205
|
const bin = detectBin();
|
|
206
|
+
const runBin = getDeployBin(bin);
|
|
206
207
|
const needDoppler = Object.values(env).some((v) => v === SENSITIVE_VALUE);
|
|
207
208
|
|
|
208
209
|
const { deploy } = createDeploy({
|
|
209
210
|
binaryName: `${getBinaryName()} deploy`,
|
|
210
211
|
name,
|
|
211
212
|
workingDirectory: detectWorkingDirectory(),
|
|
212
|
-
bin: needDoppler ? `doppler run -- ${
|
|
213
|
+
bin: needDoppler ? `doppler run -- ${runBin} run` : `${runBin} run`,
|
|
213
214
|
selector,
|
|
214
215
|
region,
|
|
215
216
|
env,
|
|
@@ -313,13 +314,14 @@ function modeIndexer<T extends IndexerStateValue, TSelector extends Selector>({
|
|
|
313
314
|
},
|
|
314
315
|
onDeploy: () => {
|
|
315
316
|
const bin = detectBin();
|
|
317
|
+
const runBin = getDeployBin(bin);
|
|
316
318
|
const needDoppler = Object.values(env).some((v) => v === SENSITIVE_VALUE);
|
|
317
319
|
|
|
318
320
|
const { deploy } = createModeDeploy({
|
|
319
321
|
binaryName: `${getBinaryName()} deploy`,
|
|
320
322
|
name,
|
|
321
323
|
workingDirectory: detectWorkingDirectory(),
|
|
322
|
-
bin: needDoppler ? `doppler run -- ${
|
|
324
|
+
bin: needDoppler ? `doppler run -- ${runBin} run` : `${runBin} run`,
|
|
323
325
|
selector,
|
|
324
326
|
region,
|
|
325
327
|
env,
|
|
@@ -437,13 +439,14 @@ function api({
|
|
|
437
439
|
},
|
|
438
440
|
onDeploy: () => {
|
|
439
441
|
const bin = detectBin();
|
|
442
|
+
const runBin = getDeployBin(bin);
|
|
440
443
|
const needDoppler = Object.values(env).some((v) => v === SENSITIVE_VALUE);
|
|
441
444
|
|
|
442
445
|
const { deploy } = createDeploy({
|
|
443
446
|
binaryName: `${getBinaryName()} deploy`,
|
|
444
447
|
name,
|
|
445
448
|
workingDirectory: detectWorkingDirectory(),
|
|
446
|
-
bin: needDoppler ? `doppler run -- ${
|
|
449
|
+
bin: needDoppler ? `doppler run -- ${runBin} run` : `${runBin} run`,
|
|
447
450
|
selector,
|
|
448
451
|
region,
|
|
449
452
|
env,
|
package/src/cli.ts
CHANGED
|
@@ -45,4 +45,9 @@ function detectBin() {
|
|
|
45
45
|
return process.argv[1].slice(wd.length + path.sep.length).replace(path.sep, "/");
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
-
|
|
48
|
+
// Run ts files via bun so Nomad/Doppler don't try to exec the file directly.
|
|
49
|
+
function getDeployBin(bin: string): string {
|
|
50
|
+
return bin.endsWith(".ts") ? `bun ${bin}` : bin;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export { getBinaryName, detectSkynetDirectory, detectWorkingDirectory, detectBin, getDeployBin };
|