datagrok-tools 6.1.0 → 6.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/commands/config.js +42 -6
- package/bin/commands/docker-gen.js +16 -0
- package/bin/commands/help.js +39 -24
- package/bin/commands/publish.js +196 -52
- package/bin/commands/test.js +540 -5
- package/bin/grok.js +4 -0
- package/bin/utils/python-celery-gen.js +247 -0
- package/bin/utils/test-utils.js +8 -4
- package/package.json +1 -1
package/bin/commands/config.js
CHANGED
|
@@ -19,6 +19,15 @@ const confTemplate = _jsYaml.default.load(_fs.default.readFileSync(confTemplateD
|
|
|
19
19
|
}));
|
|
20
20
|
const grokDir = _path.default.join(_os.default.homedir(), '.grok');
|
|
21
21
|
const confPath = _path.default.join(grokDir, 'config.yaml');
|
|
22
|
+
function defaultRegistry(url) {
|
|
23
|
+
try {
|
|
24
|
+
const hostname = new URL(url).hostname;
|
|
25
|
+
if (hostname === 'localhost' || /^[\d.]+$/.test(hostname) || hostname === '::1') return '';
|
|
26
|
+
return `registry.${hostname}`;
|
|
27
|
+
} catch (e) {
|
|
28
|
+
return '';
|
|
29
|
+
}
|
|
30
|
+
}
|
|
22
31
|
function validateKey(key) {
|
|
23
32
|
if (!key || /^([A-Za-z\d-])+$/.test(key)) return true;else return 'Developer key may only include letters, numbers, or hyphens';
|
|
24
33
|
}
|
|
@@ -33,7 +42,7 @@ function generateKeyQ(server, url) {
|
|
|
33
42
|
if (server.startsWith('local')) question.message = `Developer key for ${origin} (press ENTER to skip):`;
|
|
34
43
|
return question;
|
|
35
44
|
}
|
|
36
|
-
async function addNewServer(config) {
|
|
45
|
+
async function addNewServer(config, askRegistry = false) {
|
|
37
46
|
while (true) {
|
|
38
47
|
const addServer = (await _inquirer.default.prompt({
|
|
39
48
|
name: 'add-server',
|
|
@@ -53,17 +62,29 @@ async function addNewServer(config) {
|
|
|
53
62
|
message: 'Enter a URL:'
|
|
54
63
|
}))['server-url'];
|
|
55
64
|
const key = (await _inquirer.default.prompt(generateKeyQ(name, url)))[name];
|
|
56
|
-
|
|
65
|
+
const entry = {
|
|
57
66
|
url,
|
|
58
67
|
key
|
|
59
68
|
};
|
|
69
|
+
if (askRegistry) {
|
|
70
|
+
const regDefault = defaultRegistry(url);
|
|
71
|
+
const reg = (await _inquirer.default.prompt({
|
|
72
|
+
name: 'registry',
|
|
73
|
+
type: 'input',
|
|
74
|
+
message: `Docker registry for ${name}:`,
|
|
75
|
+
default: regDefault || undefined
|
|
76
|
+
}))['registry'] || '';
|
|
77
|
+
if (reg) entry.registry = reg;
|
|
78
|
+
}
|
|
79
|
+
config.servers[name] = entry;
|
|
60
80
|
} else break;
|
|
61
81
|
}
|
|
62
82
|
}
|
|
63
83
|
function config(args) {
|
|
64
84
|
const nOptions = Object.keys(args).length - 1;
|
|
65
|
-
const
|
|
66
|
-
const
|
|
85
|
+
const askRegistry = args.registry != null;
|
|
86
|
+
const interactiveMode = args['_'].length === 1 && (nOptions < 1 || nOptions === 1 && (args.reset || askRegistry) || nOptions === 2 && args.reset && askRegistry);
|
|
87
|
+
const hasAddServerCommand = args['_'].length === 2 && args['_'][1] === 'add' && args.server && args.key && args.k && args.alias && nOptions >= 4 && nOptions <= 6;
|
|
67
88
|
if (!interactiveMode && !hasAddServerCommand) return false;
|
|
68
89
|
if (!_fs.default.existsSync(grokDir)) _fs.default.mkdirSync(grokDir);
|
|
69
90
|
if (!_fs.default.existsSync(confPath) || args.reset) _fs.default.writeFileSync(confPath, _jsYaml.default.dump(confTemplate));
|
|
@@ -77,10 +98,15 @@ function config(args) {
|
|
|
77
98
|
color.error('URL parsing error. Please, provide a valid server URL.');
|
|
78
99
|
return false;
|
|
79
100
|
}
|
|
80
|
-
|
|
101
|
+
const server = {
|
|
81
102
|
url: args.server,
|
|
82
103
|
key: args.key
|
|
83
104
|
};
|
|
105
|
+
if (args.registry != null) {
|
|
106
|
+
const registry = typeof args.registry === 'string' ? args.registry : defaultRegistry(args.server);
|
|
107
|
+
if (registry) server.registry = registry;
|
|
108
|
+
}
|
|
109
|
+
config.servers[args.alias] = server;
|
|
84
110
|
color.success(`Successfully added the server to ${confPath}.`);
|
|
85
111
|
console.log(`Use this command to deploy packages: grok publish ${args.alias}`);
|
|
86
112
|
if (args.default) config.default = args.alias;
|
|
@@ -110,8 +136,18 @@ function config(args) {
|
|
|
110
136
|
question.default = config['servers'][server]['key'];
|
|
111
137
|
const devKey = await _inquirer.default.prompt(question);
|
|
112
138
|
config['servers'][server]['key'] = devKey[server];
|
|
139
|
+
if (askRegistry) {
|
|
140
|
+
const regDefault = config['servers'][server]['registry'] || defaultRegistry(url);
|
|
141
|
+
const reg = (await _inquirer.default.prompt({
|
|
142
|
+
name: 'registry',
|
|
143
|
+
type: 'input',
|
|
144
|
+
message: `Docker registry for ${server}:`,
|
|
145
|
+
default: regDefault || undefined
|
|
146
|
+
}))['registry'] || '';
|
|
147
|
+
if (reg) config['servers'][server]['registry'] = reg;else delete config['servers'][server]['registry'];
|
|
148
|
+
}
|
|
113
149
|
}
|
|
114
|
-
await addNewServer(config);
|
|
150
|
+
await addNewServer(config, askRegistry);
|
|
115
151
|
const defaultServer = await _inquirer.default.prompt({
|
|
116
152
|
name: 'default-server',
|
|
117
153
|
type: 'input',
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
Object.defineProperty(exports, "__esModule", {
|
|
4
|
+
value: true
|
|
5
|
+
});
|
|
6
|
+
exports.dockerGen = dockerGen;
|
|
7
|
+
var _pythonCeleryGen = require("../utils/python-celery-gen");
|
|
8
|
+
var color = _interopRequireWildcard(require("../utils/color-utils"));
|
|
9
|
+
function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
|
|
10
|
+
async function dockerGen(args) {
|
|
11
|
+
color.setVerbose(args.verbose || args.v || false);
|
|
12
|
+
const packageDir = process.cwd();
|
|
13
|
+
const generated = (0, _pythonCeleryGen.generateCeleryArtifacts)(packageDir);
|
|
14
|
+
if (generated) color.success('Generated Celery Docker artifacts');else console.log('No python/ directory with annotated functions found');
|
|
15
|
+
return true;
|
|
16
|
+
}
|
package/bin/commands/help.js
CHANGED
|
@@ -10,19 +10,20 @@ Usage: grok <command>
|
|
|
10
10
|
Datagrok's package management tool
|
|
11
11
|
|
|
12
12
|
Commands:
|
|
13
|
-
add
|
|
14
|
-
api
|
|
15
|
-
build
|
|
16
|
-
check
|
|
17
|
-
claude
|
|
18
|
-
config
|
|
19
|
-
create
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
13
|
+
add Add an object template
|
|
14
|
+
api Create wrapper functions
|
|
15
|
+
build Build a package or multiple packages
|
|
16
|
+
check Check package content (function signatures, etc.)
|
|
17
|
+
claude Launch Claude Code in a Datagrok dev container
|
|
18
|
+
config Create and manage config files
|
|
19
|
+
create Create a package
|
|
20
|
+
docker-gen Generate Celery Docker artifacts from Python functions
|
|
21
|
+
init Modify a package template
|
|
22
|
+
link Link \`datagrok-api\` and libraries for local development
|
|
23
|
+
publish Upload a package
|
|
24
|
+
test Run package tests
|
|
25
|
+
testall Run packages tests
|
|
26
|
+
migrate Migrate legacy tags to meta.role
|
|
26
27
|
|
|
27
28
|
To get help on a particular command, use:
|
|
28
29
|
grok <command> --help
|
|
@@ -122,13 +123,14 @@ Usage: grok config
|
|
|
122
123
|
Create or update a configuration file
|
|
123
124
|
|
|
124
125
|
Options:
|
|
125
|
-
[--reset] [--server] [--alias] [-k | --key]
|
|
126
|
+
[--reset] [--server] [--alias] [-k | --key] [--registry]
|
|
126
127
|
|
|
127
128
|
--reset Restore the default config file template
|
|
128
129
|
--server Use to add a server to the config (\`grok config add --alias alias --server url --key key\`)
|
|
129
130
|
--alias Use in conjunction with the \`server\` option to set the server name
|
|
130
131
|
--key Use in conjunction with the \`server\` option to set the developer key
|
|
131
132
|
--default Use in conjunction with the \`server\` option to set the added server as default
|
|
133
|
+
--registry Docker registry URL (default: registry.{server hostname})
|
|
132
134
|
`;
|
|
133
135
|
const HELP_CREATE = `
|
|
134
136
|
Usage: grok create [name]
|
|
@@ -156,16 +158,17 @@ Uploads a package
|
|
|
156
158
|
Checks for errors before publishing — the package won't be published if there are any.
|
|
157
159
|
|
|
158
160
|
Options:
|
|
159
|
-
[--all] [--refresh] [--link] [--build] [--release] [--rebuild-docker] [--skip-check] [-v | --verbose]
|
|
160
|
-
|
|
161
|
-
--all
|
|
162
|
-
--refresh
|
|
163
|
-
--link
|
|
164
|
-
--build
|
|
165
|
-
--release
|
|
166
|
-
--rebuild-docker
|
|
167
|
-
--skip-
|
|
168
|
-
--
|
|
161
|
+
[--all] [--refresh] [--link] [--build] [--release] [--rebuild-docker] [--skip-docker-rebuild] [--skip-check] [-v | --verbose]
|
|
162
|
+
|
|
163
|
+
--all Publish all available packages (run in packages directory)
|
|
164
|
+
--refresh Publish all available already loaded packages (run in packages directory)
|
|
165
|
+
--link Link the package to local packages
|
|
166
|
+
--build Builds the package
|
|
167
|
+
--release Publish package as release version
|
|
168
|
+
--rebuild-docker Force rebuild Docker images locally before pushing to registry
|
|
169
|
+
--skip-docker-rebuild Skip auto-rebuild when Dockerfile folder has changed
|
|
170
|
+
--skip-check Skip check stage
|
|
171
|
+
--verbose Print detailed output
|
|
169
172
|
|
|
170
173
|
Running \`grok publish\` is the same as running \`grok publish defaultHost --build --debug\`
|
|
171
174
|
`;
|
|
@@ -267,6 +270,17 @@ Options:
|
|
|
267
270
|
--verbose Prints detailed information about linked packages
|
|
268
271
|
--all Links all available packages(run in packages directory)
|
|
269
272
|
`;
|
|
273
|
+
const HELP_DOCKER_GEN = `
|
|
274
|
+
Usage: grok docker-gen
|
|
275
|
+
|
|
276
|
+
Generate Celery Docker artifacts from annotated Python functions in the python/ directory.
|
|
277
|
+
Produces Dockerfile, tasks.yaml, and Celery entry point in dockerfiles/<name>/.
|
|
278
|
+
|
|
279
|
+
Options:
|
|
280
|
+
[-v | --verbose]
|
|
281
|
+
|
|
282
|
+
--verbose Print detailed output
|
|
283
|
+
`;
|
|
270
284
|
const HELP_MIGRATE = `
|
|
271
285
|
Usage: grok migrate
|
|
272
286
|
|
|
@@ -315,6 +329,7 @@ const help = exports.help = {
|
|
|
315
329
|
claude: HELP_CLAUDE,
|
|
316
330
|
config: HELP_CONFIG,
|
|
317
331
|
create: HELP_CREATE,
|
|
332
|
+
'docker-gen': HELP_DOCKER_GEN,
|
|
318
333
|
init: HELP_INIT,
|
|
319
334
|
link: HELP_LINK,
|
|
320
335
|
publish: HELP_PUBLISH,
|
package/bin/commands/publish.js
CHANGED
|
@@ -7,6 +7,7 @@ Object.defineProperty(exports, "__esModule", {
|
|
|
7
7
|
exports.processPackage = processPackage;
|
|
8
8
|
exports.publish = publish;
|
|
9
9
|
var _archiverPromise = _interopRequireDefault(require("archiver-promise"));
|
|
10
|
+
var _crypto = _interopRequireDefault(require("crypto"));
|
|
10
11
|
var _fs = _interopRequireDefault(require("fs"));
|
|
11
12
|
var _nodeFetch = _interopRequireDefault(require("node-fetch"));
|
|
12
13
|
var _os = _interopRequireDefault(require("os"));
|
|
@@ -17,6 +18,7 @@ var _testUtils = require("../utils/test-utils");
|
|
|
17
18
|
var utils = _interopRequireWildcard(require("../utils/utils"));
|
|
18
19
|
var color = _interopRequireWildcard(require("../utils/color-utils"));
|
|
19
20
|
var _check = require("./check");
|
|
21
|
+
var _pythonCeleryGen = require("../utils/python-celery-gen");
|
|
20
22
|
function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
|
|
21
23
|
// @ts-ignore
|
|
22
24
|
|
|
@@ -37,7 +39,7 @@ const packDir = _path.default.join(curDir, 'package.json');
|
|
|
37
39
|
const packageFiles = ['src/package.ts', 'src/detectors.ts', 'src/package.js', 'src/detectors.js', 'src/package-test.ts', 'src/package-test.js', 'package.js', 'detectors.js'];
|
|
38
40
|
let config;
|
|
39
41
|
const BLEEDING_EDGE_TAG = 'bleeding-edge';
|
|
40
|
-
function discoverDockerfiles(packageName, version) {
|
|
42
|
+
function discoverDockerfiles(packageName, version, debug) {
|
|
41
43
|
const dockerfilesDir = _path.default.join(curDir, 'dockerfiles');
|
|
42
44
|
if (!_fs.default.existsSync(dockerfilesDir)) return [];
|
|
43
45
|
const results = [];
|
|
@@ -50,11 +52,12 @@ function discoverDockerfiles(packageName, version) {
|
|
|
50
52
|
if (!_fs.default.existsSync(dockerfilePath)) continue;
|
|
51
53
|
const cleanName = utils.removeScope(packageName).toLowerCase();
|
|
52
54
|
const imageName = `${cleanName}-${entry.name.toLowerCase()}`;
|
|
55
|
+
const imageTag = debug ? version : `${version}.X`;
|
|
53
56
|
results.push({
|
|
54
57
|
imageName,
|
|
55
|
-
imageTag
|
|
58
|
+
imageTag,
|
|
56
59
|
dirName: entry.name,
|
|
57
|
-
fullLocalName: `${imageName}:${
|
|
60
|
+
fullLocalName: `${imageName}:${imageTag}`
|
|
58
61
|
});
|
|
59
62
|
}
|
|
60
63
|
return results;
|
|
@@ -93,7 +96,10 @@ function dockerTag(source, target) {
|
|
|
93
96
|
}
|
|
94
97
|
function dockerPush(image) {
|
|
95
98
|
try {
|
|
96
|
-
|
|
99
|
+
execSync(`docker push ${image}`, {
|
|
100
|
+
encoding: 'utf-8',
|
|
101
|
+
stdio: 'inherit'
|
|
102
|
+
});
|
|
97
103
|
return true;
|
|
98
104
|
} catch (e) {
|
|
99
105
|
color.error(`Failed to push ${image}: ${e.message || e}`);
|
|
@@ -105,7 +111,7 @@ function dockerBuild(imageName, dockerfilePath, context) {
|
|
|
105
111
|
color.log(`Building Docker image ${imageName}...`);
|
|
106
112
|
execSync(`docker build -t ${imageName} -f ${dockerfilePath} ${context}`, {
|
|
107
113
|
encoding: 'utf-8',
|
|
108
|
-
stdio:
|
|
114
|
+
stdio: 'inherit'
|
|
109
115
|
});
|
|
110
116
|
return true;
|
|
111
117
|
} catch (e) {
|
|
@@ -113,50 +119,171 @@ function dockerBuild(imageName, dockerfilePath, context) {
|
|
|
113
119
|
return false;
|
|
114
120
|
}
|
|
115
121
|
}
|
|
116
|
-
|
|
122
|
+
function calculateFolderHash(dirPath) {
|
|
123
|
+
const hash = _crypto.default.createHash('sha256');
|
|
124
|
+
const entries = listRecursive(dirPath, '');
|
|
125
|
+
entries.sort((a, b) => a.relPath < b.relPath ? -1 : a.relPath > b.relPath ? 1 : 0);
|
|
126
|
+
for (const entry of entries) {
|
|
127
|
+
if (entry.isDir) {
|
|
128
|
+
hash.update(`dir:${entry.relPath}:`);
|
|
129
|
+
} else {
|
|
130
|
+
hash.update(`file:${entry.relPath}:`);
|
|
131
|
+
hash.update(_fs.default.readFileSync(entry.fullPath));
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
return hash.digest('hex');
|
|
135
|
+
}
|
|
136
|
+
function listRecursive(basePath, rel) {
|
|
137
|
+
const results = [];
|
|
138
|
+
const fullDir = rel ? _path.default.join(basePath, rel) : basePath;
|
|
139
|
+
for (const entry of _fs.default.readdirSync(fullDir, {
|
|
140
|
+
withFileTypes: true
|
|
141
|
+
})) {
|
|
142
|
+
const relPath = rel ? `${rel}/${entry.name}` : entry.name;
|
|
143
|
+
const fullPath = _path.default.join(fullDir, entry.name);
|
|
144
|
+
if (entry.isDirectory()) {
|
|
145
|
+
results.push({
|
|
146
|
+
relPath,
|
|
147
|
+
fullPath,
|
|
148
|
+
isDir: true
|
|
149
|
+
});
|
|
150
|
+
results.push(...listRecursive(basePath, relPath));
|
|
151
|
+
} else results.push({
|
|
152
|
+
relPath,
|
|
153
|
+
fullPath,
|
|
154
|
+
isDir: false
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
return results;
|
|
158
|
+
}
|
|
159
|
+
async function getUserLogin(host, devKey) {
|
|
160
|
+
let loginResp;
|
|
117
161
|
try {
|
|
118
|
-
|
|
119
|
-
const loginResp = await (0, _nodeFetch.default)(`${host}/users/login/dev/${devKey}`, {
|
|
162
|
+
loginResp = await (0, _nodeFetch.default)(`${host}/users/login/dev/${devKey}`, {
|
|
120
163
|
method: 'POST'
|
|
121
164
|
});
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
165
|
+
} catch (e) {
|
|
166
|
+
color.warn(`Cannot reach server ${host}: ${e.message || e}`);
|
|
167
|
+
return null;
|
|
168
|
+
}
|
|
169
|
+
if (loginResp.status !== 200) return null;
|
|
170
|
+
const loginData = await loginResp.json();
|
|
171
|
+
const token = loginData.token;
|
|
172
|
+
try {
|
|
173
|
+
const userResp = await (0, _nodeFetch.default)(`${host}/users/current`, {
|
|
127
174
|
headers: {
|
|
128
175
|
'Authorization': token
|
|
129
176
|
}
|
|
130
177
|
});
|
|
131
|
-
if (
|
|
132
|
-
|
|
133
|
-
|
|
178
|
+
if (userResp.status !== 200) return null;
|
|
179
|
+
const userData = await userResp.json();
|
|
180
|
+
return {
|
|
181
|
+
login: userData.login,
|
|
182
|
+
token
|
|
183
|
+
};
|
|
184
|
+
} catch (e) {
|
|
134
185
|
return null;
|
|
135
186
|
}
|
|
136
187
|
}
|
|
137
|
-
async function
|
|
138
|
-
const
|
|
188
|
+
async function resolveLatestCompatible(host, devKey, dockerName, version, contentHash) {
|
|
189
|
+
const userInfo = await getUserLogin(host, devKey);
|
|
190
|
+
if (!userInfo) return {
|
|
191
|
+
found: null,
|
|
192
|
+
serverError: `Authentication failed. Check your developer key.`
|
|
193
|
+
};
|
|
194
|
+
try {
|
|
195
|
+
let url = `${host}/docker/images/${encodeURIComponent(dockerName)}/latest-compatible`;
|
|
196
|
+
const params = [];
|
|
197
|
+
if (version) params.push(`version=${encodeURIComponent(version)}`);
|
|
198
|
+
if (contentHash) params.push(`contentHash=${encodeURIComponent(contentHash)}`);
|
|
199
|
+
if (params.length > 0) url += '?' + params.join('&');
|
|
200
|
+
const resp = await (0, _nodeFetch.default)(url, {
|
|
201
|
+
headers: {
|
|
202
|
+
'Authorization': userInfo.token
|
|
203
|
+
}
|
|
204
|
+
});
|
|
205
|
+
if (resp.status === 200) return {
|
|
206
|
+
found: await resp.json(),
|
|
207
|
+
serverError: null
|
|
208
|
+
};
|
|
209
|
+
if (resp.status === 404) return {
|
|
210
|
+
found: null,
|
|
211
|
+
serverError: null
|
|
212
|
+
};
|
|
213
|
+
return {
|
|
214
|
+
found: null,
|
|
215
|
+
serverError: `Unexpected response (HTTP ${resp.status}) from latest-compatible endpoint`
|
|
216
|
+
};
|
|
217
|
+
} catch (e) {
|
|
218
|
+
return {
|
|
219
|
+
found: null,
|
|
220
|
+
serverError: `Failed to query latest-compatible: ${e.message || e}`
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
async function processDockerImages(packageName, version, registry, devKey, host, rebuildDocker, zip, localTimestamps, debug, skipDockerRebuild = false) {
|
|
225
|
+
const dockerImages = discoverDockerfiles(packageName, version, debug);
|
|
139
226
|
if (dockerImages.length === 0) return;
|
|
140
227
|
color.log(`Found ${dockerImages.length} Dockerfile(s)`);
|
|
141
228
|
if (registry) dockerLogin(registry, devKey);
|
|
142
229
|
for (const img of dockerImages) {
|
|
230
|
+
color.info(`Processing docker image ${img.fullLocalName}...`);
|
|
143
231
|
let result;
|
|
232
|
+
const dockerfileDir = _path.default.join('dockerfiles', img.dirName);
|
|
233
|
+
const dockerfilePath = _path.default.join(dockerfileDir, 'Dockerfile');
|
|
234
|
+
const contentHash = calculateFolderHash(_path.default.join(curDir, dockerfileDir));
|
|
144
235
|
if (rebuildDocker) {
|
|
145
|
-
const dockerfileDir = _path.default.join('dockerfiles', img.dirName);
|
|
146
|
-
const dockerfilePath = _path.default.join(dockerfileDir, 'Dockerfile');
|
|
147
236
|
if (dockerBuild(img.fullLocalName, dockerfilePath, dockerfileDir)) {
|
|
148
|
-
result = pushImage(img, registry
|
|
237
|
+
result = pushImage(img, registry);
|
|
149
238
|
color.success(`Built and tagged ${img.fullLocalName}`);
|
|
150
239
|
} else {
|
|
151
|
-
result = await fallbackImage(img, host, devKey, registry);
|
|
240
|
+
result = await fallbackImage(img, host, devKey, registry, version, contentHash);
|
|
152
241
|
}
|
|
153
242
|
} else if (localImageExists(img.fullLocalName)) {
|
|
154
|
-
result = pushImage(img, registry, version);
|
|
155
243
|
color.success(`Found local image ${img.fullLocalName}`);
|
|
244
|
+
result = pushImage(img, registry);
|
|
156
245
|
} else {
|
|
157
|
-
|
|
158
|
-
color.
|
|
159
|
-
|
|
246
|
+
color.warn(`Local image not found. Expected: ${img.fullLocalName}`);
|
|
247
|
+
color.log(` Build it with: docker build -t ${img.fullLocalName} -f ${dockerfilePath} ${dockerfileDir}`);
|
|
248
|
+
const fallback = await fallbackImage(img, host, devKey, registry, version, contentHash);
|
|
249
|
+
if (fallback.serverError) {
|
|
250
|
+
color.error(`Cannot resolve fallback: ${fallback.serverError}`);
|
|
251
|
+
result = {
|
|
252
|
+
image: null,
|
|
253
|
+
fallback: true,
|
|
254
|
+
requestedVersion: img.imageTag
|
|
255
|
+
};
|
|
256
|
+
} else if (fallback.image && fallback.hashMatch === true) {
|
|
257
|
+
result = fallback;
|
|
258
|
+
color.success(`Falling back to ${fallback.image} (dockerfile unchanged)`);
|
|
259
|
+
} else if (fallback.image && fallback.hashMatch === false && !skipDockerRebuild) {
|
|
260
|
+
color.warn(`Dockerfile folder has changed. Rebuilding image...`);
|
|
261
|
+
if (dockerBuild(img.fullLocalName, dockerfilePath, dockerfileDir)) {
|
|
262
|
+
result = pushImage(img, registry);
|
|
263
|
+
color.success(`Built and tagged ${img.fullLocalName}`);
|
|
264
|
+
} else {
|
|
265
|
+
result = {
|
|
266
|
+
image: fallback.image,
|
|
267
|
+
fallback: true,
|
|
268
|
+
requestedVersion: img.imageTag
|
|
269
|
+
};
|
|
270
|
+
color.warn(`Build failed. Falling back to ${fallback.image} (hash mismatch)`);
|
|
271
|
+
}
|
|
272
|
+
} else {
|
|
273
|
+
// No fallback and no local image — must build
|
|
274
|
+
color.warn(`No fallback available. Building ${img.fullLocalName}...`);
|
|
275
|
+
if (dockerBuild(img.fullLocalName, dockerfilePath, dockerfileDir)) {
|
|
276
|
+
result = pushImage(img, registry);
|
|
277
|
+
color.success(`Built and tagged ${img.fullLocalName}`);
|
|
278
|
+
} else {
|
|
279
|
+
result = {
|
|
280
|
+
image: null,
|
|
281
|
+
fallback: true,
|
|
282
|
+
requestedVersion: img.imageTag
|
|
283
|
+
};
|
|
284
|
+
color.error(`Failed to build ${img.fullLocalName}. No container will be available.`);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
160
287
|
}
|
|
161
288
|
const imageJsonPath = _path.default.join('dockerfiles', img.dirName, 'image.json');
|
|
162
289
|
zip.append(JSON.stringify(result, null, 2), {
|
|
@@ -166,56 +293,66 @@ async function processDockerImages(packageName, version, registry, devKey, host,
|
|
|
166
293
|
color.log(`Added ${imageJsonPath}`);
|
|
167
294
|
}
|
|
168
295
|
}
|
|
169
|
-
function pushImage(img, registry
|
|
296
|
+
function pushImage(img, registry) {
|
|
297
|
+
const canonicalImage = `datagrok/${img.fullLocalName}`;
|
|
170
298
|
if (registry) {
|
|
171
|
-
const remoteTag = `${registry}/${
|
|
299
|
+
const remoteTag = `${registry}/${canonicalImage}`;
|
|
172
300
|
dockerTag(img.fullLocalName, remoteTag);
|
|
173
301
|
if (dockerPush(remoteTag)) return {
|
|
174
|
-
image:
|
|
302
|
+
image: canonicalImage,
|
|
175
303
|
fallback: false
|
|
176
304
|
};
|
|
177
305
|
color.warn(`Push failed, image tagged locally only: ${remoteTag}`);
|
|
178
306
|
return {
|
|
179
|
-
image:
|
|
307
|
+
image: canonicalImage,
|
|
180
308
|
fallback: false
|
|
181
309
|
};
|
|
182
310
|
}
|
|
183
311
|
color.warn('No registry configured. Image tagged locally only.');
|
|
184
312
|
return {
|
|
185
|
-
image:
|
|
313
|
+
image: canonicalImage,
|
|
186
314
|
fallback: false
|
|
187
315
|
};
|
|
188
316
|
}
|
|
189
|
-
async function fallbackImage(img, host, devKey, registry) {
|
|
190
|
-
const
|
|
191
|
-
|
|
192
|
-
|
|
317
|
+
async function fallbackImage(img, host, devKey, registry, version, contentHash) {
|
|
318
|
+
const {
|
|
319
|
+
found,
|
|
320
|
+
serverError
|
|
321
|
+
} = await resolveLatestCompatible(host, devKey, img.imageName, version, contentHash);
|
|
322
|
+
if (serverError) return {
|
|
323
|
+
image: null,
|
|
193
324
|
fallback: true,
|
|
194
|
-
requestedVersion: img.imageTag
|
|
325
|
+
requestedVersion: img.imageTag,
|
|
326
|
+
serverError
|
|
327
|
+
};
|
|
328
|
+
if (found) return {
|
|
329
|
+
image: found.image,
|
|
330
|
+
fallback: true,
|
|
331
|
+
requestedVersion: img.imageTag,
|
|
332
|
+
hashMatch: found.hashMatch
|
|
195
333
|
};
|
|
196
|
-
color.warn('No previous version available. Container will not be available until an image is built.');
|
|
197
334
|
return {
|
|
198
335
|
image: null,
|
|
199
336
|
fallback: true,
|
|
200
337
|
requestedVersion: img.imageTag
|
|
201
338
|
};
|
|
202
339
|
}
|
|
203
|
-
async function processPackage(debug, rebuild, host, devKey, packageName, dropDb, suffix, hostAlias, registry, rebuildDocker) {
|
|
204
|
-
//
|
|
340
|
+
async function processPackage(debug, rebuild, host, devKey, packageName, dropDb, suffix, hostAlias, registry, rebuildDocker, skipDockerRebuild) {
|
|
341
|
+
// Validate server connectivity and dev key
|
|
205
342
|
let timestamps = {};
|
|
206
343
|
let url = `${host}/packages/dev/${devKey}/${packageName}`;
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
return 1;
|
|
213
|
-
}
|
|
214
|
-
} catch (error) {
|
|
215
|
-
if (utils.isConnectivityError(error)) color.error(`Server is possibly offline: ${host}`);
|
|
216
|
-
if (color.isVerbose()) console.error(error);
|
|
344
|
+
try {
|
|
345
|
+
const checkResp = await (0, _nodeFetch.default)(url + '/timestamps');
|
|
346
|
+
const checkData = await checkResp.json();
|
|
347
|
+
if (checkData['#type'] === 'ApiError') {
|
|
348
|
+
color.error(checkData.message);
|
|
217
349
|
return 1;
|
|
218
350
|
}
|
|
351
|
+
if (debug) timestamps = checkData;
|
|
352
|
+
} catch (error) {
|
|
353
|
+
if (utils.isConnectivityError(error)) color.error(`Server is possibly offline: ${host}`);
|
|
354
|
+
if (color.isVerbose()) console.error(error);
|
|
355
|
+
return 1;
|
|
219
356
|
}
|
|
220
357
|
const zip = (0, _archiverPromise.default)('zip', {
|
|
221
358
|
store: false
|
|
@@ -313,9 +450,16 @@ async function processPackage(debug, rebuild, host, devKey, packageName, dropDb,
|
|
|
313
450
|
return 1;
|
|
314
451
|
}
|
|
315
452
|
|
|
453
|
+
// Generate Celery Docker artifacts from python/ if present
|
|
454
|
+
(0, _pythonCeleryGen.generateCeleryArtifacts)(curDir);
|
|
455
|
+
|
|
316
456
|
// Process Docker images and inject image.json into the ZIP
|
|
317
|
-
|
|
318
|
-
|
|
457
|
+
let dockerVersion = json.version;
|
|
458
|
+
if (debug) {
|
|
459
|
+
const userInfo = await getUserLogin(host, devKey);
|
|
460
|
+
if (userInfo) dockerVersion = userInfo.login;
|
|
461
|
+
}
|
|
462
|
+
await processDockerImages(packageName, dockerVersion, registry, devKey, host, rebuildDocker ?? false, zip, localTimestamps, debug, skipDockerRebuild ?? false);
|
|
319
463
|
zip.append(JSON.stringify(localTimestamps), {
|
|
320
464
|
name: 'timestamps.json'
|
|
321
465
|
});
|
|
@@ -460,7 +604,7 @@ async function publishPackage(args) {
|
|
|
460
604
|
if (!args.suffix && stdout) args.suffix = stdout.toString().substring(0, 8);
|
|
461
605
|
});
|
|
462
606
|
await utils.delay(100);
|
|
463
|
-
code = await processPackage(!args.release, Boolean(args.rebuild), url, key, packageName, args.dropDb ?? false, args.suffix, host, registry, args['rebuild-docker']);
|
|
607
|
+
code = await processPackage(!args.release, Boolean(args.rebuild), url, key, packageName, args.dropDb ?? false, args.suffix, host, registry, args['rebuild-docker'], args['skip-docker-rebuild']);
|
|
464
608
|
} catch (error) {
|
|
465
609
|
console.error(error);
|
|
466
610
|
code = 1;
|