m8flow 1.0.2 → 1.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. package/bundled/backend/Dockerfile +41 -0
  2. package/bundled/backend/add_nodes.py +416 -0
  3. package/bundled/backend/api/routes/appstate.py +102 -0
  4. package/bundled/backend/api/routes/flows.py +64 -5
  5. package/bundled/backend/api/routes/nodes.py +25 -1
  6. package/bundled/backend/core/code_validator.py +2 -0
  7. package/bundled/backend/core/executor.py +19 -3
  8. package/bundled/backend/main.py +16 -4
  9. package/bundled/backend/requirements.txt +27 -6
  10. package/bundled/backend/services/llm_service.py +984 -108
  11. package/bundled/backend/services/self_healer.py +1 -1
  12. package/bundled/backend/temp.json +0 -0
  13. package/bundled/backend/templates.json +0 -0
  14. package/bundled/backend/templates.py +2907 -745
  15. package/bundled/backend/warmup.py +65 -0
  16. package/bundled/frontend-dist/assets/index-CKUZ27n8.css +1 -0
  17. package/bundled/frontend-dist/assets/index-DNaB6zf0.js +46 -0
  18. package/bundled/frontend-dist/index.html +2 -2
  19. package/lib/backend.js +184 -35
  20. package/lib/ports.js +42 -0
  21. package/lib/run.js +42 -15
  22. package/lib/setup.js +143 -59
  23. package/package.json +5 -4
  24. package/scripts/check-docker.js +35 -0
  25. package/bundled/frontend-dist/assets/index-BAQ3lKsy.css +0 -1
  26. package/bundled/frontend-dist/assets/index-CZCCzeUC.js +0 -41
package/bundled/frontend-dist/index.html CHANGED
@@ -8,8 +8,8 @@
8
8
  <link rel="preconnect" href="https://fonts.googleapis.com" />
9
9
  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
10
10
  <title>M8Flow — ML Pipeline Builder</title>
11
- <script type="module" crossorigin src="/assets/index-CZCCzeUC.js"></script>
12
- <link rel="stylesheet" crossorigin href="/assets/index-BAQ3lKsy.css">
11
+ <script type="module" crossorigin src="/assets/index-DNaB6zf0.js"></script>
12
+ <link rel="stylesheet" crossorigin href="/assets/index-CKUZ27n8.css">
13
13
  </head>
14
14
  <body>
15
15
  <div id="root"></div>
package/lib/backend.js CHANGED
@@ -1,22 +1,157 @@
1
1
  /**
2
2
  * Backend process manager.
3
- * Uses execa to spawn uvicorn and streams filtered output to the terminal.
3
+ *
4
+ * Strategy (in priority order):
5
+ * 1. Docker image m8flow-backend:latest exists → docker run (warm, instant)
6
+ * 2. Docker available but image missing → docker build then run (~5 min first time)
7
+ * 3. No Docker → fall back to Python venv + uvicorn (existing behaviour)
4
8
  */
5
- import { execa } from 'execa';
6
- import path from 'path';
7
- import { log } from './logger.js';
9
+ import { execa } from 'execa';
10
+ import path from 'path';
11
+ import fs from 'fs';
12
+ import { log } from './logger.js';
13
+ import { fileURLToPath } from 'url';
8
14
 
9
- // Uvicorn log lines we suppress (just noise in a CLI context)
10
- const SUPPRESS = /^\s*(INFO|DEBUG):\s*(Started|Waiting|Application startup|Uvicorn running)/i;
15
+ const __dirname = path.dirname(fileURLToPath(import.meta.url));
11
16
 
12
- /**
13
- * Spawns the FastAPI backend.
14
- * Returns the execa ChildProcess — caller is responsible for .kill().
15
- */
16
- export function spawnBackend(venvPython, backendDir, port, m8flowHome, verbose) {
17
+ const DOCKER_IMAGE = 'm8flow-backend:latest';
18
+ const SUPPRESS_VENV = /^\s*(INFO|DEBUG):\s*(Started|Waiting|Application startup|Uvicorn running)/i;
19
+
20
+ // ── Docker helpers ────────────────────────────────────────────────────────────
21
+
22
+ let _dockerAvailableCache = null; // null = not checked yet
23
+
24
+ async function isDockerAvailable(m8flowHome) {
25
+ // Cache in memory for the process lifetime — `docker info` takes 1-2 s
26
+ if (_dockerAvailableCache !== null) return _dockerAvailableCache;
27
+
28
+ // Also cache to disk so repeated CLI invocations skip the check for 24 h
29
+ if (m8flowHome) {
30
+ const flagFile = path.join(m8flowHome, '.docker_available');
31
+ try {
32
+ const stat = fs.statSync(flagFile);
33
+ const ageMs = Date.now() - stat.mtimeMs;
34
+ if (ageMs < 86_400_000) { // 24-hour TTL
35
+ _dockerAvailableCache = fs.readFileSync(flagFile, 'utf8').trim() === '1';
36
+ return _dockerAvailableCache;
37
+ }
38
+ } catch {}
39
+ }
40
+
41
+ try {
42
+ const { exitCode } = await execa('docker', ['info'], { reject: false, timeout: 4_000 });
43
+ _dockerAvailableCache = exitCode === 0;
44
+ } catch {
45
+ _dockerAvailableCache = false;
46
+ }
47
+
48
+ if (m8flowHome) {
49
+ try {
50
+ fs.writeFileSync(path.join(m8flowHome, '.docker_available'), _dockerAvailableCache ? '1' : '0');
51
+ } catch {}
52
+ }
53
+
54
+ return _dockerAvailableCache;
55
+ }
56
+
57
+ async function isImageBuilt() {
58
+ try {
59
+ const { stdout } = await execa(
60
+ 'docker', ['image', 'inspect', DOCKER_IMAGE, '--format', '{{.Id}}'],
61
+ { reject: false, timeout: 5_000 },
62
+ );
63
+ return stdout.trim().length > 0;
64
+ } catch {
65
+ return false;
66
+ }
67
+ }
68
+
69
+ async function buildImage(backendDir) {
70
+ log.info(`Building Docker image ${DOCKER_IMAGE} (one-time, ~5 min)…`);
71
+ log.dim('All ML packages will be pre-baked — subsequent starts will be instant.');
72
+
73
+ const proc = execa('docker', ['build', '-t', DOCKER_IMAGE, backendDir], {
74
+ stdio: 'inherit',
75
+ timeout: 600_000, // 10-minute cap
76
+ });
77
+
78
+ try {
79
+ await proc;
80
+ log.ok(`Image ${DOCKER_IMAGE} built — future starts will be instant.`);
81
+ return true;
82
+ } catch (err) {
83
+ log.error(`Docker build failed: ${err.message}`);
84
+ return false;
85
+ }
86
+ }
87
+
88
+ // ── Docker backend ────────────────────────────────────────────────────────────
89
+
90
+ function spawnDockerBackend(port, m8flowHome, verbose) {
91
+ const uploadsDir = path.join(m8flowHome, 'uploads');
92
+ const modelsDir = path.join(m8flowHome, 'models');
93
+ const pipelinesDir = path.join(m8flowHome, 'pipelines');
94
+
95
+ const args = [
96
+ 'run', '--rm',
97
+ '-p', `127.0.0.1:${port}:8000`,
98
+ '-v', `${uploadsDir}:/data/uploads`,
99
+ '-v', `${modelsDir}:/data/models`,
100
+ '-v', `${pipelinesDir}:/data/pipelines`,
101
+ '-e', 'M8FLOW_ENV=docker',
102
+ '-e', `M8FLOW_UPLOAD_DIR=/data/uploads`,
103
+ '-e', `M8FLOW_MODELS_DIR=/data/models`,
104
+ '-e', `M8FLOW_PIPELINE_DIR=/data/pipelines`,
105
+ // Forward OpenRouter key if set on host
106
+ ...(process.env.OPENROUTER_API_KEY
107
+ ? ['-e', `OPENROUTER_API_KEY=${process.env.OPENROUTER_API_KEY}`]
108
+ : []),
109
+ '--name', 'm8flow_backend',
110
+ DOCKER_IMAGE,
111
+ ];
112
+
113
+ const proc = execa('docker', args, {
114
+ stdout: 'pipe',
115
+ stderr: 'pipe',
116
+ reject: false,
117
+ cleanup: true,
118
+ });
119
+
120
+ proc.stdout.on('data', (chunk) => {
121
+ if (verbose) process.stdout.write(` [docker] ${chunk}`);
122
+ });
123
+
124
+ proc.stderr.on('data', (chunk) => {
125
+ const text = chunk.toString();
126
+ if (verbose) {
127
+ process.stderr.write(` [docker] ${text}`);
128
+ } else {
129
+ for (const line of text.split('\n')) {
130
+ const l = line.trim();
131
+ if (l && (l.includes('ERROR') || l.includes('CRITICAL'))) {
132
+ log.error(`[api] ${l}`);
133
+ }
134
+ }
135
+ }
136
+ });
137
+
138
+ proc.on('exit', (code) => {
139
+ if (code !== 0 && code !== null) {
140
+ log.error(`Docker backend exited (code ${code}).`);
141
+ }
142
+ });
143
+
144
+ return proc;
145
+ }
146
+
147
+ // ── Venv backend (fallback) ───────────────────────────────────────────────────
148
+
149
+ function spawnVenvBackend(venvPython, backendDir, port, m8flowHome, verbose) {
17
150
  const env = {
18
151
  ...process.env,
19
152
  PYTHONUNBUFFERED: '1',
153
+ // Use the pre-compiled .pyc cache built by warmup.py — cuts import time by ~70%
154
+ PYTHONPYCACHEPREFIX: path.join(m8flowHome, 'pycache'),
20
155
  M8FLOW_UPLOAD_DIR: path.join(m8flowHome, 'uploads'),
21
156
  M8FLOW_MODELS_DIR: path.join(m8flowHome, 'models'),
22
157
  M8FLOW_PIPELINE_DIR: path.join(m8flowHome, 'pipelines'),
@@ -24,22 +159,9 @@ export function spawnBackend(venvPython, backendDir, port, m8flowHome, verbose)
24
159
 
25
160
  const proc = execa(
26
161
  venvPython,
27
- [
28
- '-m', 'uvicorn',
29
- 'main:app',
30
- '--host', '127.0.0.1',
31
- '--port', String(port),
32
- '--log-level', verbose ? 'info' : 'warning',
33
- ],
34
- {
35
- cwd: backendDir,
36
- env,
37
- // Don't inherit stdio — we handle output ourselves
38
- stdout: 'pipe',
39
- stderr: 'pipe',
40
- reject: false, // don't throw on non-zero exit
41
- cleanup: true, // execa kills the child on parent exit
42
- }
162
+ ['-m', 'uvicorn', 'main:app', '--host', '127.0.0.1', '--port', String(port),
163
+ '--log-level', verbose ? 'info' : 'warning'],
164
+ { cwd: backendDir, env, stdout: 'pipe', stderr: 'pipe', reject: false, cleanup: true },
43
165
  );
44
166
 
45
167
  proc.stdout.on('data', (chunk) => {
@@ -51,10 +173,9 @@ export function spawnBackend(venvPython, backendDir, port, m8flowHome, verbose)
51
173
  if (verbose) {
52
174
  process.stderr.write(prefixLines('[api] ', chunk));
53
175
  } else {
54
- // Even in quiet mode, surface real errors
55
176
  for (const line of text.split('\n')) {
56
177
  const l = line.trim();
57
- if (l && !SUPPRESS.test(l) && (l.includes('ERROR') || l.includes('CRITICAL'))) {
178
+ if (l && !SUPPRESS_VENV.test(l) && (l.includes('ERROR') || l.includes('CRITICAL'))) {
58
179
  log.error(`[api] ${l}`);
59
180
  }
60
181
  }
@@ -71,11 +192,39 @@ export function spawnBackend(venvPython, backendDir, port, m8flowHome, verbose)
71
192
  return proc;
72
193
  }
73
194
 
195
+ // ── Public API ────────────────────────────────────────────────────────────────
196
+
197
+ /**
198
+ * Spawn the backend using Docker if available (warm, instant) or venv (cold).
199
+ * Returns { proc, mode } where mode is 'docker' | 'venv'.
200
+ */
201
+ export async function spawnBackend(venvPython, backendDir, port, m8flowHome, verbose) {
202
+ // ── Try Docker first ──────────────────────────────────────────────────────
203
+ const dockerAvailable = await isDockerAvailable(m8flowHome);
204
+
205
+ if (dockerAvailable) {
206
+ // Stop any leftover container from a previous run
207
+ await execa('docker', ['rm', '-f', 'm8flow_backend'], { reject: false }).catch(() => {});
208
+
209
+ const imageExists = await isImageBuilt();
210
+ if (!imageExists) {
211
+ log.warn(`Docker image ${DOCKER_IMAGE} not found — building now (one-time, ~5 min)…`);
212
+ const built = await buildImage(backendDir);
213
+ if (!built) {
214
+ log.warn('Docker build failed — falling back to Python venv.');
215
+ return { proc: spawnVenvBackend(venvPython, backendDir, port, m8flowHome, verbose), mode: 'venv' };
216
+ }
217
+ }
218
+
219
+ log.ok(`Using Docker backend (${DOCKER_IMAGE}) — environment is warm.`);
220
+ return { proc: spawnDockerBackend(port, m8flowHome, verbose), mode: 'docker' };
221
+ }
222
+
223
+ // ── Docker not available — use venv ───────────────────────────────────────
224
+ log.dim('Docker not found — using Python venv backend.');
225
+ return { proc: spawnVenvBackend(venvPython, backendDir, port, m8flowHome, verbose), mode: 'venv' };
226
+ }
227
+
74
228
  function prefixLines(prefix, chunk) {
75
- return chunk
76
- .toString()
77
- .split('\n')
78
- .filter(Boolean)
79
- .map((l) => ` ${prefix}${l}\n`)
80
- .join('');
229
+ return chunk.toString().split('\n').filter(Boolean).map((l) => ` ${prefix}${l}\n`).join('');
81
230
  }
package/lib/ports.js CHANGED
@@ -3,6 +3,48 @@
3
3
  */
4
4
  import detectPort from 'detect-port';
5
5
  import net from 'net';
6
+ import { execa } from 'execa';
7
+
8
+ /**
9
+ * Kill whatever process is listening on `port`.
10
+ * Used to clean up stale m8flow backends from previous sessions that
11
+ * didn't get properly terminated (common on Windows after a forced close).
12
+ *
13
+ * Cross-platform:
14
+ * Windows → netstat -ano + taskkill /F /T
15
+ * Mac/Linux → lsof + kill -9
16
+ */
17
+ export async function killProcessOnPort(port) {
18
+ try {
19
+ if (process.platform === 'win32') {
20
+ // netstat output: TCP 127.0.0.1:8000 0.0.0.0:0 LISTENING 12345
21
+ const { stdout = '' } = await execa('netstat', ['-ano'], {
22
+ reject: false, timeout: 4_000,
23
+ });
24
+ for (const line of stdout.split('\n')) {
25
+ // Match lines with our port in LISTENING state
26
+ if (!line.includes(`:${port}`) || !line.toUpperCase().includes('LISTEN')) continue;
27
+ const parts = line.trim().split(/\s+/);
28
+ const pid = parts[parts.length - 1];
29
+ if (!pid || pid === '0' || !/^\d+$/.test(pid)) continue;
30
+ // /T = kill entire process tree, /F = force
31
+ await execa('taskkill', ['/PID', pid, '/F', '/T'], { reject: false });
32
+ await sleep(600); // let the OS release the port
33
+ return;
34
+ }
35
+ } else {
36
+ // lsof -ti:<port> lists PIDs listening on that port
37
+ await execa('sh', ['-c', `lsof -ti:${port} | xargs kill -9 2>/dev/null; true`], {
38
+ reject: false, timeout: 4_000,
39
+ });
40
+ await sleep(400);
41
+ }
42
+ } catch {
43
+ // Non-fatal — if we can't kill it, detect-port will find a different port
44
+ }
45
+ }
46
+
47
+ function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
6
48
 
7
49
  /**
8
50
  * Returns a free port starting from `preferred`.
package/lib/run.js CHANGED
@@ -13,16 +13,17 @@
13
13
  * 9. Open browser
14
14
  * 10. Keep alive until Ctrl+C
15
15
  */
16
- import path from 'path';
17
- import os from 'os';
18
- import fs from 'fs';
16
+ import path from 'path';
17
+ import os from 'os';
18
+ import fs from 'fs';
19
+ import { execSync } from 'child_process';
19
20
  import { createRequire } from 'module';
20
21
  import { fileURLToPath } from 'url';
21
22
 
22
23
  import { checkPython, ensureVenv, installDeps } from './setup.js';
23
24
  import { spawnBackend } from './backend.js';
24
25
  import { serveStatic } from './server.js';
25
- import { resolvePort, waitForPort } from './ports.js';
26
+ import { resolvePort, waitForPort, killProcessOnPort } from './ports.js';
26
27
  import { banner, log, spin, readyBox } from './logger.js';
27
28
 
28
29
  const require = createRequire(import.meta.url);
@@ -67,21 +68,35 @@ export async function run({
67
68
  // ── 4. pip install ────────────────────────────────────────────────────────
68
69
  await installDeps(venvPython, BACKEND_DIR, M8FLOW_HOME);
69
70
 
70
- // ── 5. Ports ──────────────────────────────────────────────────────────────
71
+ // ── 5. Ports — kill stale backends first ────────────────────────────────────
71
72
  log.section('Network');
73
+
74
+ // Kill any leftover process from a previous session BEFORE checking port availability.
75
+ // On Windows, Ctrl+C sometimes leaves the Python/uvicorn process running.
76
+ await killProcessOnPort(backendPort);
77
+ await killProcessOnPort(frontendPort);
78
+
72
79
  const fe = await resolvePort(frontendPort, 'frontend');
73
80
  const be = await resolvePort(backendPort, 'backend');
74
81
 
75
- if (fe.changed) log.warn(`Port ${fe.from} busy → using ${fe.port} for the frontend`);
76
- if (be.changed) log.warn(`Port ${be.from} busy → using ${be.port} for the backend`);
82
+ if (fe.changed) log.warn(`Port ${fe.from} still busy after cleanup → using ${fe.port}`);
83
+ if (be.changed) log.warn(`Port ${be.from} still busy after cleanup → using ${be.port}`);
77
84
 
78
85
  log.ok(`Frontend → http://localhost:${fe.port}`);
79
86
  log.ok(`Backend → http://localhost:${be.port}`);
80
87
 
81
- // ── 6. Backend ────────────────────────────────────────────────────────────
88
+ // ── 6. Backend (Docker-first, venv fallback) ─────────────────────────────
82
89
  log.section('Starting servers');
83
- const sp6 = spin('Starting FastAPI backend…');
84
- const backendProc = spawnBackend(venvPython, BACKEND_DIR, be.port, M8FLOW_HOME, verbose);
90
+ const sp6 = spin('Launching backend…');
91
+ const { proc: backendProc, mode: backendMode } = await spawnBackend(
92
+ venvPython, BACKEND_DIR, be.port, M8FLOW_HOME, verbose
93
+ );
94
+
95
+ if (backendMode === 'docker') {
96
+ sp6.update('Docker backend starting (warm — no cold-start)…');
97
+ } else {
98
+ sp6.update('Starting FastAPI backend (venv)…');
99
+ }
85
100
 
86
101
  backendProc.on('exit', (code) => {
87
102
  if (code !== 0 && code !== null) process.exit(1); // propagate crash
@@ -90,12 +105,15 @@ export async function run({
90
105
  // ── 7. Frontend ───────────────────────────────────────────────────────────
91
106
  const httpServer = await serveStatic(FRONTEND_DIR, fe.port);
92
107
 
93
- // ── 8. Health-check backend ───────────────────────────────────────────────
108
+ // ── 8. Health-check backend ─────────────────────────────────────────────────
109
+ // 120 s gives Docker image pull + container start enough headroom on first run.
110
+ // The frontend shows its own "Initializing" overlay during this window.
94
111
  sp6.update('Waiting for backend to accept connections…');
95
- await waitForPort(be.port).catch(() => {
96
- log.warn('Backend is taking longer than usual — check --verbose for details');
112
+ await waitForPort(be.port, '127.0.0.1', 120_000).catch(() => {
113
+ log.warn('Backend is taking longer than expected — the UI will retry automatically.');
97
114
  });
98
- sp6.succeed('Backend is up');
115
+ sp6.succeed(`Backend is up (${backendMode === 'docker' ? '🐳 Docker — warm' : 'Python venv'})`);
116
+ console.log();
99
117
 
100
118
  // ── 9. Open browser ───────────────────────────────────────────────────────
101
119
  const appUrl = `http://localhost:${fe.port}`;
@@ -116,7 +134,16 @@ export async function run({
116
134
  const shutdown = (signal) => {
117
135
  log.blank();
118
136
  log.info(`Received ${signal} — shutting down…`);
119
- try { backendProc.kill(); } catch {}
137
+
138
+ // On Windows, .kill() only kills the execa wrapper — the Python subprocess
139
+ // often survives, leaving the port busy on the next run.
140
+ // Use taskkill /T to kill the entire process tree.
141
+ if (process.platform === 'win32' && backendProc.pid) {
142
+ try { execSync(`taskkill /PID ${backendProc.pid} /F /T`, { stdio: 'ignore' }); } catch {}
143
+ } else {
144
+ try { backendProc.kill('SIGTERM'); } catch {}
145
+ }
146
+
120
147
  try { httpServer.close(); } catch {}
121
148
  log.ok('Goodbye!\n');
122
149
  process.exit(0);