m8flow 1.0.1 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundled/backend/Dockerfile +41 -0
- package/bundled/backend/add_nodes.py +416 -0
- package/bundled/backend/api/routes/appstate.py +102 -0
- package/bundled/backend/api/routes/flows.py +64 -5
- package/bundled/backend/api/routes/nodes.py +25 -1
- package/bundled/backend/core/code_validator.py +2 -0
- package/bundled/backend/core/executor.py +19 -3
- package/bundled/backend/main.py +16 -4
- package/bundled/backend/requirements.txt +27 -6
- package/bundled/backend/services/llm_service.py +957 -98
- package/bundled/backend/services/self_healer.py +1 -1
- package/bundled/backend/storage/__init__.py +0 -0
- package/bundled/backend/storage/memory.py +16 -0
- package/bundled/backend/temp.json +0 -0
- package/bundled/backend/templates.json +0 -0
- package/bundled/backend/templates.py +2907 -745
- package/bundled/backend/warmup.py +65 -0
- package/bundled/frontend-dist/assets/index-CKUZ27n8.css +1 -0
- package/bundled/frontend-dist/assets/index-DNaB6zf0.js +46 -0
- package/bundled/frontend-dist/index.html +2 -2
- package/lib/backend.js +155 -35
- package/lib/run.js +18 -7
- package/lib/setup.js +119 -59
- package/package.json +3 -2
- package/scripts/build.js +1 -2
- package/scripts/check-docker.js +35 -0
- package/bundled/frontend-dist/assets/index-Dm2J6DQp.js +0 -41
- package/bundled/frontend-dist/assets/index-xKOV3MGm.css +0 -1
|
@@ -8,8 +8,8 @@
|
|
|
8
8
|
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
|
9
9
|
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
|
10
10
|
<title>M8Flow — ML Pipeline Builder</title>
|
|
11
|
-
<script type="module" crossorigin src="/assets/index-
|
|
12
|
-
<link rel="stylesheet" crossorigin href="/assets/index-
|
|
11
|
+
<script type="module" crossorigin src="/assets/index-DNaB6zf0.js"></script>
|
|
12
|
+
<link rel="stylesheet" crossorigin href="/assets/index-CKUZ27n8.css">
|
|
13
13
|
</head>
|
|
14
14
|
<body>
|
|
15
15
|
<div id="root"></div>
|
package/lib/backend.js
CHANGED
|
@@ -1,19 +1,125 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Backend process manager.
|
|
3
|
-
*
|
|
3
|
+
*
|
|
4
|
+
* Strategy (in priority order):
|
|
5
|
+
* 1. Docker image m8flow-backend:latest exists → docker run (warm, instant)
|
|
6
|
+
* 2. Docker available but image missing → docker build then run (~5 min first time)
|
|
7
|
+
* 3. No Docker → fall back to Python venv + uvicorn (existing behaviour)
|
|
4
8
|
*/
|
|
5
|
-
import { execa }
|
|
6
|
-
import path
|
|
7
|
-
import { log }
|
|
9
|
+
import { execa } from 'execa';
|
|
10
|
+
import path from 'path';
|
|
11
|
+
import { log } from './logger.js';
|
|
12
|
+
import { fileURLToPath } from 'url';
|
|
8
13
|
|
|
9
|
-
|
|
10
|
-
const SUPPRESS = /^\s*(INFO|DEBUG):\s*(Started|Waiting|Application startup|Uvicorn running)/i;
|
|
14
|
+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
11
15
|
|
|
12
|
-
|
|
13
|
-
*
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
16
|
+
const DOCKER_IMAGE = 'm8flow-backend:latest';
|
|
17
|
+
const SUPPRESS_VENV = /^\s*(INFO|DEBUG):\s*(Started|Waiting|Application startup|Uvicorn running)/i;
|
|
18
|
+
|
|
19
|
+
// ── Docker helpers ────────────────────────────────────────────────────────────
|
|
20
|
+
|
|
21
|
+
async function isDockerAvailable() {
|
|
22
|
+
try {
|
|
23
|
+
const { exitCode } = await execa('docker', ['info'], { reject: false, timeout: 5_000 });
|
|
24
|
+
return exitCode === 0;
|
|
25
|
+
} catch {
|
|
26
|
+
return false;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
async function isImageBuilt() {
|
|
31
|
+
try {
|
|
32
|
+
const { stdout } = await execa(
|
|
33
|
+
'docker', ['image', 'inspect', DOCKER_IMAGE, '--format', '{{.Id}}'],
|
|
34
|
+
{ reject: false, timeout: 5_000 },
|
|
35
|
+
);
|
|
36
|
+
return stdout.trim().length > 0;
|
|
37
|
+
} catch {
|
|
38
|
+
return false;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
async function buildImage(backendDir) {
|
|
43
|
+
log.info(`Building Docker image ${DOCKER_IMAGE} (one-time, ~5 min)…`);
|
|
44
|
+
log.dim('All ML packages will be pre-baked — subsequent starts will be instant.');
|
|
45
|
+
|
|
46
|
+
const proc = execa('docker', ['build', '-t', DOCKER_IMAGE, backendDir], {
|
|
47
|
+
stdio: 'inherit',
|
|
48
|
+
timeout: 600_000, // 10-minute cap
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
try {
|
|
52
|
+
await proc;
|
|
53
|
+
log.ok(`Image ${DOCKER_IMAGE} built — future starts will be instant.`);
|
|
54
|
+
return true;
|
|
55
|
+
} catch (err) {
|
|
56
|
+
log.error(`Docker build failed: ${err.message}`);
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// ── Docker backend ────────────────────────────────────────────────────────────
|
|
62
|
+
|
|
63
|
+
function spawnDockerBackend(port, m8flowHome, verbose) {
|
|
64
|
+
const uploadsDir = path.join(m8flowHome, 'uploads');
|
|
65
|
+
const modelsDir = path.join(m8flowHome, 'models');
|
|
66
|
+
const pipelinesDir = path.join(m8flowHome, 'pipelines');
|
|
67
|
+
|
|
68
|
+
const args = [
|
|
69
|
+
'run', '--rm',
|
|
70
|
+
'-p', `127.0.0.1:${port}:8000`,
|
|
71
|
+
'-v', `${uploadsDir}:/data/uploads`,
|
|
72
|
+
'-v', `${modelsDir}:/data/models`,
|
|
73
|
+
'-v', `${pipelinesDir}:/data/pipelines`,
|
|
74
|
+
'-e', 'M8FLOW_ENV=docker',
|
|
75
|
+
'-e', `M8FLOW_UPLOAD_DIR=/data/uploads`,
|
|
76
|
+
'-e', `M8FLOW_MODELS_DIR=/data/models`,
|
|
77
|
+
'-e', `M8FLOW_PIPELINE_DIR=/data/pipelines`,
|
|
78
|
+
// Forward OpenRouter key if set on host
|
|
79
|
+
...(process.env.OPENROUTER_API_KEY
|
|
80
|
+
? ['-e', `OPENROUTER_API_KEY=${process.env.OPENROUTER_API_KEY}`]
|
|
81
|
+
: []),
|
|
82
|
+
'--name', 'm8flow_backend',
|
|
83
|
+
DOCKER_IMAGE,
|
|
84
|
+
];
|
|
85
|
+
|
|
86
|
+
const proc = execa('docker', args, {
|
|
87
|
+
stdout: 'pipe',
|
|
88
|
+
stderr: 'pipe',
|
|
89
|
+
reject: false,
|
|
90
|
+
cleanup: true,
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
proc.stdout.on('data', (chunk) => {
|
|
94
|
+
if (verbose) process.stdout.write(` [docker] ${chunk}`);
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
proc.stderr.on('data', (chunk) => {
|
|
98
|
+
const text = chunk.toString();
|
|
99
|
+
if (verbose) {
|
|
100
|
+
process.stderr.write(` [docker] ${text}`);
|
|
101
|
+
} else {
|
|
102
|
+
for (const line of text.split('\n')) {
|
|
103
|
+
const l = line.trim();
|
|
104
|
+
if (l && (l.includes('ERROR') || l.includes('CRITICAL'))) {
|
|
105
|
+
log.error(`[api] ${l}`);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
proc.on('exit', (code) => {
|
|
112
|
+
if (code !== 0 && code !== null) {
|
|
113
|
+
log.error(`Docker backend exited (code ${code}).`);
|
|
114
|
+
}
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
return proc;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// ── Venv backend (fallback) ───────────────────────────────────────────────────
|
|
121
|
+
|
|
122
|
+
function spawnVenvBackend(venvPython, backendDir, port, m8flowHome, verbose) {
|
|
17
123
|
const env = {
|
|
18
124
|
...process.env,
|
|
19
125
|
PYTHONUNBUFFERED: '1',
|
|
@@ -24,22 +130,9 @@ export function spawnBackend(venvPython, backendDir, port, m8flowHome, verbose)
|
|
|
24
130
|
|
|
25
131
|
const proc = execa(
|
|
26
132
|
venvPython,
|
|
27
|
-
[
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
'--host', '127.0.0.1',
|
|
31
|
-
'--port', String(port),
|
|
32
|
-
'--log-level', verbose ? 'info' : 'warning',
|
|
33
|
-
],
|
|
34
|
-
{
|
|
35
|
-
cwd: backendDir,
|
|
36
|
-
env,
|
|
37
|
-
// Don't inherit stdio — we handle output ourselves
|
|
38
|
-
stdout: 'pipe',
|
|
39
|
-
stderr: 'pipe',
|
|
40
|
-
reject: false, // don't throw on non-zero exit
|
|
41
|
-
cleanup: true, // execa kills the child on parent exit
|
|
42
|
-
}
|
|
133
|
+
['-m', 'uvicorn', 'main:app', '--host', '127.0.0.1', '--port', String(port),
|
|
134
|
+
'--log-level', verbose ? 'info' : 'warning'],
|
|
135
|
+
{ cwd: backendDir, env, stdout: 'pipe', stderr: 'pipe', reject: false, cleanup: true },
|
|
43
136
|
);
|
|
44
137
|
|
|
45
138
|
proc.stdout.on('data', (chunk) => {
|
|
@@ -51,10 +144,9 @@ export function spawnBackend(venvPython, backendDir, port, m8flowHome, verbose)
|
|
|
51
144
|
if (verbose) {
|
|
52
145
|
process.stderr.write(prefixLines('[api] ', chunk));
|
|
53
146
|
} else {
|
|
54
|
-
// Even in quiet mode, surface real errors
|
|
55
147
|
for (const line of text.split('\n')) {
|
|
56
148
|
const l = line.trim();
|
|
57
|
-
if (l && !
|
|
149
|
+
if (l && !SUPPRESS_VENV.test(l) && (l.includes('ERROR') || l.includes('CRITICAL'))) {
|
|
58
150
|
log.error(`[api] ${l}`);
|
|
59
151
|
}
|
|
60
152
|
}
|
|
@@ -71,11 +163,39 @@ export function spawnBackend(venvPython, backendDir, port, m8flowHome, verbose)
|
|
|
71
163
|
return proc;
|
|
72
164
|
}
|
|
73
165
|
|
|
166
|
+
// ── Public API ────────────────────────────────────────────────────────────────
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* Spawn the backend using Docker if available (warm, instant) or venv (cold).
|
|
170
|
+
* Returns { proc, mode } where mode is 'docker' | 'venv'.
|
|
171
|
+
*/
|
|
172
|
+
export async function spawnBackend(venvPython, backendDir, port, m8flowHome, verbose) {
|
|
173
|
+
// ── Try Docker first ──────────────────────────────────────────────────────
|
|
174
|
+
const dockerAvailable = await isDockerAvailable();
|
|
175
|
+
|
|
176
|
+
if (dockerAvailable) {
|
|
177
|
+
// Stop any leftover container from a previous run
|
|
178
|
+
await execa('docker', ['rm', '-f', 'm8flow_backend'], { reject: false }).catch(() => {});
|
|
179
|
+
|
|
180
|
+
const imageExists = await isImageBuilt();
|
|
181
|
+
if (!imageExists) {
|
|
182
|
+
log.warn(`Docker image ${DOCKER_IMAGE} not found — building now (one-time, ~5 min)…`);
|
|
183
|
+
const built = await buildImage(backendDir);
|
|
184
|
+
if (!built) {
|
|
185
|
+
log.warn('Docker build failed — falling back to Python venv.');
|
|
186
|
+
return { proc: spawnVenvBackend(venvPython, backendDir, port, m8flowHome, verbose), mode: 'venv' };
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
log.ok(`Using Docker backend (${DOCKER_IMAGE}) — environment is warm.`);
|
|
191
|
+
return { proc: spawnDockerBackend(port, m8flowHome, verbose), mode: 'docker' };
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// ── Docker not available — use venv ───────────────────────────────────────
|
|
195
|
+
log.dim('Docker not found — using Python venv backend.');
|
|
196
|
+
return { proc: spawnVenvBackend(venvPython, backendDir, port, m8flowHome, verbose), mode: 'venv' };
|
|
197
|
+
}
|
|
198
|
+
|
|
74
199
|
function prefixLines(prefix, chunk) {
|
|
75
|
-
return chunk
|
|
76
|
-
.toString()
|
|
77
|
-
.split('\n')
|
|
78
|
-
.filter(Boolean)
|
|
79
|
-
.map((l) => ` ${prefix}${l}\n`)
|
|
80
|
-
.join('');
|
|
200
|
+
return chunk.toString().split('\n').filter(Boolean).map((l) => ` ${prefix}${l}\n`).join('');
|
|
81
201
|
}
|
package/lib/run.js
CHANGED
|
@@ -78,10 +78,18 @@ export async function run({
|
|
|
78
78
|
log.ok(`Frontend → http://localhost:${fe.port}`);
|
|
79
79
|
log.ok(`Backend → http://localhost:${be.port}`);
|
|
80
80
|
|
|
81
|
-
// ── 6. Backend
|
|
81
|
+
// ── 6. Backend (Docker-first, venv fallback) ─────────────────────────────
|
|
82
82
|
log.section('Starting servers');
|
|
83
|
-
const sp6 = spin('
|
|
84
|
-
const
|
|
83
|
+
const sp6 = spin('Launching backend…');
|
|
84
|
+
const { proc: backendProc, mode: backendMode } = await spawnBackend(
|
|
85
|
+
venvPython, BACKEND_DIR, be.port, M8FLOW_HOME, verbose
|
|
86
|
+
);
|
|
87
|
+
|
|
88
|
+
if (backendMode === 'docker') {
|
|
89
|
+
sp6.update('Docker backend starting (warm — no cold-start)…');
|
|
90
|
+
} else {
|
|
91
|
+
sp6.update('Starting FastAPI backend (venv)…');
|
|
92
|
+
}
|
|
85
93
|
|
|
86
94
|
backendProc.on('exit', (code) => {
|
|
87
95
|
if (code !== 0 && code !== null) process.exit(1); // propagate crash
|
|
@@ -90,12 +98,15 @@ export async function run({
|
|
|
90
98
|
// ── 7. Frontend ───────────────────────────────────────────────────────────
|
|
91
99
|
const httpServer = await serveStatic(FRONTEND_DIR, fe.port);
|
|
92
100
|
|
|
93
|
-
// ── 8. Health-check backend
|
|
101
|
+
// ── 8. Health-check backend ─────────────────────────────────────────────────
|
|
102
|
+
// 120 s gives Docker image pull + container start enough headroom on first run.
|
|
103
|
+
// The frontend shows its own "Initializing" overlay during this window.
|
|
94
104
|
sp6.update('Waiting for backend to accept connections…');
|
|
95
|
-
await waitForPort(be.port).catch(() => {
|
|
96
|
-
log.warn('Backend is taking longer than
|
|
105
|
+
await waitForPort(be.port, '127.0.0.1', 120_000).catch(() => {
|
|
106
|
+
log.warn('Backend is taking longer than expected — the UI will retry automatically.');
|
|
97
107
|
});
|
|
98
|
-
sp6.succeed(
|
|
108
|
+
sp6.succeed(`Backend is up (${backendMode === 'docker' ? '🐳 Docker — warm' : 'Python venv'})`);
|
|
109
|
+
console.log();
|
|
99
110
|
|
|
100
111
|
// ── 9. Open browser ───────────────────────────────────────────────────────
|
|
101
112
|
const appUrl = `http://localhost:${fe.port}`;
|
package/lib/setup.js
CHANGED
|
@@ -1,13 +1,16 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Python environment setup
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
2
|
+
* Python environment setup — with a real-time progress bar during pip install.
|
|
3
|
+
*
|
|
4
|
+
* checkPython() → finds Python 3.8+
|
|
5
|
+
* ensureVenv() → creates ~/.m8flow/venv (auto-repairs broken Windows venvs)
|
|
6
|
+
* installDeps() → pip installs requirements.txt with a live progress bar
|
|
7
|
+
* (skip when hash unchanged — warm starts are instant)
|
|
6
8
|
*/
|
|
7
9
|
import { execa } from 'execa';
|
|
8
10
|
import fs from 'fs';
|
|
9
11
|
import path from 'path';
|
|
10
12
|
import crypto from 'crypto';
|
|
13
|
+
import chalk from 'chalk';
|
|
11
14
|
import { spin } from './logger.js';
|
|
12
15
|
|
|
13
16
|
// ── Python detection ──────────────────────────────────────────────────────────
|
|
@@ -42,32 +45,19 @@ export async function checkPython() {
|
|
|
42
45
|
|
|
43
46
|
async function getPythonVersion(cmd) {
|
|
44
47
|
try {
|
|
45
|
-
const { stdout, stderr } = await execa(cmd, ['--version'], {
|
|
46
|
-
reject: false,
|
|
47
|
-
timeout: 5_000,
|
|
48
|
-
});
|
|
48
|
+
const { stdout, stderr } = await execa(cmd, ['--version'], { reject: false, timeout: 5_000 });
|
|
49
49
|
const out = (stdout + stderr).trim();
|
|
50
50
|
const m = out.match(/Python (\d+)\.(\d+)/);
|
|
51
51
|
if (!m) return null;
|
|
52
52
|
return { major: parseInt(m[1], 10), minor: parseInt(m[2], 10) };
|
|
53
|
-
} catch {
|
|
54
|
-
return null;
|
|
55
|
-
}
|
|
53
|
+
} catch { return null; }
|
|
56
54
|
}
|
|
57
55
|
|
|
58
56
|
// ── Virtual environment ───────────────────────────────────────────────────────
|
|
59
57
|
|
|
60
|
-
/**
|
|
61
|
-
* On Windows, creating a venv with --upgrade-deps can produce a broken pip
|
|
62
|
-
* where certifi's cacert.pem is missing, making every HTTPS request fail.
|
|
63
|
-
* We detect and auto-delete such venvs so a clean one gets created.
|
|
64
|
-
*/
|
|
65
58
|
function isVenvBroken(venvDir) {
|
|
66
|
-
// Only a problem on Windows — cacert.pem goes missing after pip upgrade
|
|
67
59
|
if (process.platform !== 'win32') return false;
|
|
68
|
-
const cacert = path.join(
|
|
69
|
-
venvDir, 'Lib', 'site-packages', 'pip', '_vendor', 'certifi', 'cacert.pem'
|
|
70
|
-
);
|
|
60
|
+
const cacert = path.join(venvDir, 'Lib', 'site-packages', 'pip', '_vendor', 'certifi', 'cacert.pem');
|
|
71
61
|
return !fs.existsSync(cacert);
|
|
72
62
|
}
|
|
73
63
|
|
|
@@ -75,11 +65,9 @@ export async function ensureVenv(pythonCmd, m8flowHome) {
|
|
|
75
65
|
const venvDir = path.join(m8flowHome, 'venv');
|
|
76
66
|
const venvPython = venvPythonPath(venvDir);
|
|
77
67
|
|
|
78
|
-
// Auto-repair: delete corrupted venv so a fresh one gets created
|
|
79
68
|
if (fs.existsSync(venvPython) && isVenvBroken(venvDir)) {
|
|
80
69
|
const sp = spin('Detected broken virtual environment — recreating…');
|
|
81
70
|
fs.rmSync(venvDir, { recursive: true, force: true });
|
|
82
|
-
// Invalidate deps hash so pip install re-runs
|
|
83
71
|
const hashFile = path.join(m8flowHome, '.deps_hash');
|
|
84
72
|
if (fs.existsSync(hashFile)) fs.rmSync(hashFile);
|
|
85
73
|
sp.warn('Old venv removed — creating a fresh one');
|
|
@@ -88,68 +76,140 @@ export async function ensureVenv(pythonCmd, m8flowHome) {
|
|
|
88
76
|
if (!fs.existsSync(venvPython)) {
|
|
89
77
|
const sp = spin('Creating Python virtual environment…');
|
|
90
78
|
try {
|
|
91
|
-
|
|
92
|
-
// On Windows, upgrading pip inside the venv strips certifi's CA bundle,
|
|
93
|
-
// breaking every subsequent HTTPS request from pip.
|
|
94
|
-
await execa(pythonCmd, ['-m', 'venv', venvDir], {
|
|
95
|
-
timeout: 120_000,
|
|
96
|
-
});
|
|
79
|
+
await execa(pythonCmd, ['-m', 'venv', venvDir], { timeout: 120_000 });
|
|
97
80
|
sp.succeed('Virtual environment created');
|
|
98
81
|
} catch (err) {
|
|
99
82
|
sp.fail('Failed to create virtual environment');
|
|
100
|
-
throw new Error(
|
|
101
|
-
`venv creation failed: ${err.message}\n\n` +
|
|
102
|
-
` Try manually:\n ${pythonCmd} -m venv ${venvDir}`
|
|
103
|
-
);
|
|
83
|
+
throw new Error(`venv creation failed: ${err.message}\n\n Try: ${pythonCmd} -m venv ${venvDir}`);
|
|
104
84
|
}
|
|
105
85
|
}
|
|
106
86
|
|
|
107
87
|
return venvPython;
|
|
108
88
|
}
|
|
109
89
|
|
|
110
|
-
// ──
|
|
90
|
+
// ── Progress bar renderer ─────────────────────────────────────────────────────
|
|
91
|
+
|
|
92
|
+
const BAR_WIDTH = 28;
|
|
93
|
+
|
|
94
|
+
function renderBar(done, total, label = '') {
|
|
95
|
+
const pct = total > 0 ? Math.min(done / total, 1) : 0;
|
|
96
|
+
const filled = Math.round(pct * BAR_WIDTH);
|
|
97
|
+
const empty = BAR_WIDTH - filled;
|
|
98
|
+
const bar = chalk.hex('#6366f1')('█'.repeat(filled)) + chalk.dim('░'.repeat(empty));
|
|
99
|
+
const pctStr = String(Math.round(pct * 100)).padStart(3);
|
|
100
|
+
const pkg = label ? chalk.dim(` ${label.slice(0, 30)}`) : '';
|
|
101
|
+
process.stdout.write(`\r ${chalk.bold('[M8Flow]')} ${bar} ${chalk.bold(pctStr)}%${pkg} `);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function clearBar() {
|
|
105
|
+
process.stdout.write('\r' + ' '.repeat(80) + '\r');
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// ── pip install with live progress ───────────────────────────────────────────
|
|
111
109
|
|
|
112
110
|
export async function installDeps(venvPython, backendDir, m8flowHome) {
|
|
113
111
|
const reqFile = path.join(backendDir, 'requirements.txt');
|
|
114
112
|
const hashFile = path.join(m8flowHome, '.deps_hash');
|
|
115
113
|
|
|
116
|
-
if (!fs.existsSync(reqFile)) {
|
|
117
|
-
throw new Error(`requirements.txt not found at ${reqFile}`);
|
|
118
|
-
}
|
|
114
|
+
if (!fs.existsSync(reqFile)) throw new Error(`requirements.txt not found at ${reqFile}`);
|
|
119
115
|
|
|
120
116
|
const hash = sha256(fs.readFileSync(reqFile));
|
|
121
117
|
|
|
122
118
|
if (fs.existsSync(hashFile) && fs.readFileSync(hashFile, 'utf8').trim() === hash) {
|
|
123
|
-
|
|
119
|
+
console.log(` ✅ ${chalk.bold('ML engine is warm')} — dependencies already installed`);
|
|
120
|
+
return; // ← fast path: warm start, skip entirely
|
|
124
121
|
}
|
|
125
122
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
123
|
+
// ── Count packages for accurate progress ─────────────────────────────────
|
|
124
|
+
const pkgLines = fs.readFileSync(reqFile, 'utf8')
|
|
125
|
+
.split('\n')
|
|
126
|
+
.map(l => l.trim())
|
|
127
|
+
.filter(l => l && !l.startsWith('#') && !l.startsWith('-'));
|
|
128
|
+
const total = pkgLines.length;
|
|
129
|
+
|
|
130
|
+
console.log();
|
|
131
|
+
console.log(` ${chalk.bold(chalk.hex('#6366f1')('[M8Flow]'))} Initializing ML Engine — ${total} packages`);
|
|
132
|
+
console.log(` ${chalk.dim('Server will start automatically once all libraries are ready.')}`);
|
|
133
|
+
console.log();
|
|
134
|
+
|
|
135
|
+
renderBar(0, total, 'Starting…');
|
|
136
|
+
|
|
137
|
+
let installed = 0;
|
|
138
|
+
let currentPkg = '';
|
|
139
|
+
|
|
140
|
+
// pip with single --quiet outputs "Collecting X" and "Successfully installed X-Y"
|
|
141
|
+
const pipProc = execa(
|
|
142
|
+
venvPython,
|
|
143
|
+
[
|
|
144
|
+
'-m', 'pip', 'install',
|
|
145
|
+
'--quiet', // single quiet: shows Collecting + errors
|
|
146
|
+
'--disable-pip-version-check',
|
|
147
|
+
'--trusted-host', 'pypi.org',
|
|
148
|
+
'--trusted-host', 'files.pythonhosted.org',
|
|
149
|
+
'--trusted-host', 'pypi.python.org',
|
|
150
|
+
'-r', reqFile,
|
|
151
|
+
],
|
|
152
|
+
{
|
|
153
|
+
stdout: 'pipe',
|
|
154
|
+
stderr: 'pipe',
|
|
155
|
+
reject: false,
|
|
156
|
+
timeout: 600_000,
|
|
157
|
+
}
|
|
158
|
+
);
|
|
159
|
+
|
|
160
|
+
const handleLine = (line) => {
|
|
161
|
+
const l = line.trim();
|
|
162
|
+
if (!l) return;
|
|
163
|
+
|
|
164
|
+
// "Collecting package-name" → new package starting
|
|
165
|
+
const collecting = l.match(/^Collecting\s+([\w\-\.]+)/i);
|
|
166
|
+
if (collecting) {
|
|
167
|
+
currentPkg = collecting[1];
|
|
168
|
+
renderBar(installed, total, currentPkg);
|
|
169
|
+
return;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// "Successfully installed …" → one or more packages done
|
|
173
|
+
const done = l.match(/^Successfully installed (.+)/i);
|
|
174
|
+
if (done) {
|
|
175
|
+
// Count how many packages in this batch
|
|
176
|
+
const names = done[1].split(/\s+/).filter(Boolean);
|
|
177
|
+
installed = Math.min(installed + names.length, total);
|
|
178
|
+
currentPkg = names[names.length - 1] || '';
|
|
179
|
+
renderBar(installed, total, currentPkg);
|
|
180
|
+
}
|
|
181
|
+
};
|
|
182
|
+
|
|
183
|
+
pipProc.stdout.on('data', chunk =>
|
|
184
|
+
chunk.toString().split('\n').forEach(handleLine)
|
|
185
|
+
);
|
|
186
|
+
pipProc.stderr.on('data', chunk =>
|
|
187
|
+
chunk.toString().split('\n').forEach(line => {
|
|
188
|
+
const l = line.trim();
|
|
189
|
+
if (l && l.toLowerCase().includes('error')) {
|
|
190
|
+
// Show real errors outside the progress bar line
|
|
191
|
+
clearBar();
|
|
192
|
+
console.error(` ${chalk.red('⚠')} ${l}`);
|
|
193
|
+
}
|
|
194
|
+
})
|
|
195
|
+
);
|
|
142
196
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
197
|
+
const { exitCode } = await pipProc;
|
|
198
|
+
|
|
199
|
+
clearBar();
|
|
200
|
+
|
|
201
|
+
if (exitCode !== 0 && exitCode !== null) {
|
|
202
|
+
console.error();
|
|
147
203
|
throw new Error(
|
|
148
|
-
`pip install failed
|
|
204
|
+
`pip install failed (exit ${exitCode}).\n\n` +
|
|
149
205
|
` Manual fix:\n` +
|
|
150
|
-
` ${venvPython} -m pip install --trusted-host pypi.org
|
|
206
|
+
` ${venvPython} -m pip install --trusted-host pypi.org -r ${reqFile}`
|
|
151
207
|
);
|
|
152
208
|
}
|
|
209
|
+
|
|
210
|
+
fs.writeFileSync(hashFile, hash);
|
|
211
|
+
console.log(` ${chalk.green('✔')} ${chalk.bold('ML engine ready')} — all ${total} packages installed`);
|
|
212
|
+
console.log();
|
|
153
213
|
}
|
|
154
214
|
|
|
155
215
|
// ── Helpers ───────────────────────────────────────────────────────────────────
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "m8flow",
|
|
3
|
-
"version": "1.0
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"description": "Visual ML Pipeline Builder — run locally with one command",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"machine-learning",
|
|
@@ -35,6 +35,7 @@
|
|
|
35
35
|
"scripts": {
|
|
36
36
|
"build": "node scripts/build.js",
|
|
37
37
|
"dev": "node scripts/build.js && npm install -g . --force",
|
|
38
|
-
"prepublishOnly": "node scripts/build.js"
|
|
38
|
+
"prepublishOnly": "node scripts/build.js",
|
|
39
|
+
"postinstall": "node scripts/check-docker.js"
|
|
39
40
|
}
|
|
40
41
|
}
|
package/scripts/build.js
CHANGED
|
@@ -13,7 +13,6 @@ import { execSync } from 'child_process';
|
|
|
13
13
|
import fs from 'fs';
|
|
14
14
|
import path from 'path';
|
|
15
15
|
import { fileURLToPath } from 'url';
|
|
16
|
-
import crypto from 'crypto';
|
|
17
16
|
|
|
18
17
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
19
18
|
const CLI_DIR = path.resolve(__dirname, '..');
|
|
@@ -28,7 +27,7 @@ const FRONTEND_OUT = path.join(BUNDLED_DIR, 'frontend-dist');
|
|
|
28
27
|
// Skip these when copying the Python source
|
|
29
28
|
const BACKEND_SKIP = new Set([
|
|
30
29
|
'venv', '__pycache__', '.env', '.env.local', '.env.production',
|
|
31
|
-
'uploads', 'models', 'pipelines',
|
|
30
|
+
'uploads', 'models', 'pipelines',
|
|
32
31
|
'node_modules', '.git', '.mypy_cache', '.ruff_cache',
|
|
33
32
|
'dist', '*.pyc', '*.pyo', '*.pyd',
|
|
34
33
|
]);
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* check-docker.js — runs after `npm install -g m8flow` via the postinstall hook.
|
|
4
|
+
* Alerts the user if Docker is missing so they know what to expect.
|
|
5
|
+
*/
|
|
6
|
+
import { execSync } from 'child_process';
|
|
7
|
+
|
|
8
|
+
const GREEN = '\x1b[32m';
|
|
9
|
+
const YELLOW = '\x1b[33m';
|
|
10
|
+
const RESET = '\x1b[0m';
|
|
11
|
+
const BOLD = '\x1b[1m';
|
|
12
|
+
|
|
13
|
+
console.log();
|
|
14
|
+
|
|
15
|
+
let dockerAvailable = false;
|
|
16
|
+
try {
|
|
17
|
+
const ver = execSync('docker --version', { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
|
|
18
|
+
dockerAvailable = true;
|
|
19
|
+
console.log(` ${GREEN}✔${RESET} ${BOLD}Docker found${RESET} (${ver})`);
|
|
20
|
+
console.log(` M8Flow will use Docker for ${BOLD}zero cold-start${RESET} pipeline execution.`);
|
|
21
|
+
console.log(` First run: ${YELLOW}docker build${RESET} (~5 min, one-time).`);
|
|
22
|
+
console.log(` All future runs: instant warm start.`);
|
|
23
|
+
} catch {
|
|
24
|
+
console.log(` ${YELLOW}⚠${RESET} ${BOLD}Docker not found${RESET}`);
|
|
25
|
+
console.log(` M8Flow will use a Python venv backend (first run may be slower).`);
|
|
26
|
+
console.log(` Install Docker for a warm, zero-delay execution environment:`);
|
|
27
|
+
console.log(` ${YELLOW}https://www.docker.com/get-started${RESET}`);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
console.log();
|
|
31
|
+
console.log(` Run ${BOLD}m8flow run${RESET} to start.`);
|
|
32
|
+
console.log();
|
|
33
|
+
|
|
34
|
+
// postinstall scripts must exit 0 regardless — never block installation
|
|
35
|
+
process.exit(0);
|