@igorjpimenta/manifold 0.0.0-bootstrap.1 → 0.0.0-stage
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1583
- package/package.json +4 -49
- package/.migrations/ops/0000_futuristic_karma.sql +0 -111
- package/.migrations/ops/0001_low_maximus.sql +0 -7
- package/.migrations/ops/meta/0000_snapshot.json +0 -763
- package/.migrations/ops/meta/0001_snapshot.json +0 -173
- package/.migrations/ops/meta/_journal.json +0 -20
- package/.migrations/runtime/0000_real_boomer.sql +0 -180
- package/.migrations/runtime/0001_hot_dakota_north.sql +0 -2
- package/.migrations/runtime/0002_breezy_vertigo.sql +0 -8
- package/.migrations/runtime/0003_dashing_jack_flag.sql +0 -5
- package/.migrations/runtime/0004_panoramic_ego.sql +0 -1
- package/.migrations/runtime/0005_tidy_ben_grimm.sql +0 -10
- package/.migrations/runtime/0006_wet_nightmare.sql +0 -57
- package/.migrations/runtime/0007_ambiguous_chronomancer.sql +0 -8
- package/.migrations/runtime/0008_wealthy_thundra.sql +0 -301
- package/.migrations/runtime/0009_lying_shocker.sql +0 -96
- package/.migrations/runtime/0010_solid_phantom_reporter.sql +0 -160
- package/.migrations/runtime/0011_windy_demogoblin.sql +0 -175
- package/.migrations/runtime/0012_restore_harness_fks.sql +0 -103
- package/.migrations/runtime/meta/0000_snapshot.json +0 -1181
- package/.migrations/runtime/meta/0001_snapshot.json +0 -1209
- package/.migrations/runtime/meta/0002_snapshot.json +0 -1261
- package/.migrations/runtime/meta/0003_snapshot.json +0 -1292
- package/.migrations/runtime/meta/0004_snapshot.json +0 -1240
- package/.migrations/runtime/meta/0006_snapshot.json +0 -1310
- package/.migrations/runtime/meta/0007_snapshot.json +0 -1362
- package/.migrations/runtime/meta/0008_snapshot.json +0 -1742
- package/.migrations/runtime/meta/0009_snapshot.json +0 -2413
- package/.migrations/runtime/meta/0010_snapshot.json +0 -2562
- package/.migrations/runtime/meta/0011_snapshot.json +0 -2812
- package/.migrations/runtime/meta/0012_snapshot.json +0 -2812
- package/.migrations/runtime/meta/_journal.json +0 -97
- package/RELEASING.md +0 -89
- package/agent/fleet.js +0 -100
- package/bin/cli.js +0 -854
- package/bin/manifold.js +0 -4
- package/boot/flow-revival.js +0 -536
- package/budget/index.js +0 -495
- package/budget/lib.js +0 -59
- package/dashboard/cost.js +0 -528
- package/dashboard/dist/assets/index-CUfp63sj.css +0 -1
- package/dashboard/dist/assets/index-Qy9JMpma.js +0 -371
- package/dashboard/dist/index.html +0 -13
- package/dashboard/event-log.js +0 -267
- package/dashboard/index.html +0 -12
- package/dashboard/package-lock.json +0 -2877
- package/dashboard/package.json +0 -37
- package/dashboard/pipeline-graph.js +0 -131
- package/dashboard/postcss.config.js +0 -6
- package/dashboard/search.js +0 -183
- package/dashboard/server.js +0 -1015
- package/dashboard/src/App.tsx +0 -42
- package/dashboard/src/api/client.ts +0 -118
- package/dashboard/src/api/types.ts +0 -510
- package/dashboard/src/api/ws.tsx +0 -112
- package/dashboard/src/components/CancelButton.tsx +0 -63
- package/dashboard/src/components/ClickToCopy.tsx +0 -68
- package/dashboard/src/components/CopyButton.tsx +0 -61
- package/dashboard/src/components/DailyCostTrend.tsx +0 -85
- package/dashboard/src/components/DailyTimeTrend.tsx +0 -94
- package/dashboard/src/components/Funnel.tsx +0 -149
- package/dashboard/src/components/GateForm.tsx +0 -158
- package/dashboard/src/components/GraphView.tsx +0 -302
- package/dashboard/src/components/GroupPathsSummary.tsx +0 -47
- package/dashboard/src/components/Histogram.tsx +0 -292
- package/dashboard/src/components/Layout.tsx +0 -28
- package/dashboard/src/components/NeedsAttention.tsx +0 -98
- package/dashboard/src/components/NodeContextCard.tsx +0 -132
- package/dashboard/src/components/PathTrail.tsx +0 -35
- package/dashboard/src/components/PipelineCostBreakdown.tsx +0 -277
- package/dashboard/src/components/RerunButton.tsx +0 -46
- package/dashboard/src/components/RunComparison.tsx +0 -53
- package/dashboard/src/components/Sparkline.tsx +0 -94
- package/dashboard/src/components/StatStrip.tsx +0 -147
- package/dashboard/src/components/StateTree.tsx +0 -84
- package/dashboard/src/components/StatusBadge.tsx +0 -18
- package/dashboard/src/components/SubflowPathPills.tsx +0 -38
- package/dashboard/src/components/SubflowSummary.tsx +0 -24
- package/dashboard/src/components/SubflowsCard.tsx +0 -327
- package/dashboard/src/components/TodayStrip.tsx +0 -86
- package/dashboard/src/components/TranscriptPane.tsx +0 -99
- package/dashboard/src/components/TranscriptRow.tsx +0 -69
- package/dashboard/src/components/ui/badge.tsx +0 -38
- package/dashboard/src/components/ui/button.tsx +0 -28
- package/dashboard/src/components/ui/card.tsx +0 -42
- package/dashboard/src/components/ui/skeleton.tsx +0 -9
- package/dashboard/src/components/ui/table.tsx +0 -62
- package/dashboard/src/index.css +0 -26
- package/dashboard/src/lib/anomaly.ts +0 -42
- package/dashboard/src/lib/format.ts +0 -30
- package/dashboard/src/lib/step-history.ts +0 -13
- package/dashboard/src/lib/subflow.ts +0 -154
- package/dashboard/src/lib/use-click-outside.ts +0 -28
- package/dashboard/src/lib/use-ignored-clusters.ts +0 -67
- package/dashboard/src/lib/utils.ts +0 -6
- package/dashboard/src/main.tsx +0 -10
- package/dashboard/src/types/shims.d.ts +0 -4
- package/dashboard/src/views/Home.tsx +0 -478
- package/dashboard/src/views/PipelineDetail.tsx +0 -1185
- package/dashboard/src/views/PipelinesList.tsx +0 -111
- package/dashboard/src/views/RunDetail.tsx +0 -248
- package/dashboard/src/views/Search.tsx +0 -196
- package/dashboard/stats.js +0 -668
- package/dashboard/tailwind.config.ts +0 -43
- package/dashboard/tree-stats.js +0 -12
- package/dashboard/tsconfig.json +0 -25
- package/dashboard/views.js +0 -231
- package/dashboard/vite.config.ts +0 -34
- package/drizzle.config.js +0 -12
- package/drizzle.config.ops.js +0 -12
- package/elevation.js +0 -116
- package/engine/admission-promotion-resume.js +0 -127
- package/engine/agent-dispatch.js +0 -64
- package/engine/child-resume-trigger.js +0 -315
- package/engine/executor-budget.js +0 -87
- package/engine/executor-cancel.js +0 -83
- package/engine/executor-errors.js +0 -122
- package/engine/executor-helpers.js +0 -254
- package/engine/executor-nodes.js +0 -292
- package/engine/executor-persist.js +0 -212
- package/engine/executor-subflow-children.js +0 -275
- package/engine/executor.js +0 -1013
- package/engine/global-concurrency.js +0 -33
- package/engine/loader.js +0 -528
- package/engine/needs-resolver.js +0 -229
- package/engine/pipelines-watcher.js +0 -133
- package/engine/result-replay.js +0 -34
- package/engine/revival-messages.js +0 -11
- package/engine/spawn-child.js +0 -55
- package/engine/wait-for-flow-settlement.js +0 -109
- package/events/catalog.js +0 -26
- package/events/emit.js +0 -44
- package/events/kinds.js +0 -118
- package/events/serve-ws.js +0 -74
- package/events/stream.js +0 -35
- package/events/ws-dialog.js +0 -257
- package/events/ws-frame.js +0 -194
- package/events/ws-run-events.js +0 -70
- package/events.js +0 -32
- package/gates/gate-resolver.js +0 -576
- package/gates/paused-flows.js +0 -55
- package/graphql/columns.js +0 -67
- package/graphql/schema.js +0 -333
- package/index.js +0 -49
- package/mcp/http.js +0 -159
- package/mcp/identity.js +0 -94
- package/mcp/stdio.js +0 -80
- package/mcp/tools.js +0 -245
- package/mcp/wire-ops.js +0 -178
- package/routes/bearer-auth.js +0 -57
- package/routes/discuss-audio.js +0 -87
- package/routes/discuss-component.js +0 -39
- package/routes/discuss-draft.js +0 -159
- package/routes/discuss-helpers.js +0 -20
- package/routes/discuss.js +0 -848
- package/routes/failures.js +0 -104
- package/routes/graphql.js +0 -61
- package/routes/health.js +0 -54
- package/routes/operator-discuss.js +0 -193
- package/routes/operator-resume.js +0 -32
- package/routes/operator-retry.js +0 -301
- package/routes/operator.js +0 -539
- package/routes/pipelines.js +0 -118
- package/routes/runs.js +0 -1129
- package/scripts/ci/import-check.mjs +0 -50
- package/scripts/link-core-self.mjs +0 -31
- package/scripts/with-guard.sh +0 -24
- package/server/app.js +0 -503
- package/server/registry.js +0 -53
- package/server/serve.js +0 -163
- package/state/admission-queue-store.js +0 -325
- package/state/attribution-registries.js +0 -102
- package/state/batch-writer.js +0 -162
- package/state/db-guard.js +0 -63
- package/state/db-health.js +0 -27
- package/state/db.js +0 -122
- package/state/discuss-store.js +0 -299
- package/state/failure-store.js +0 -427
- package/state/migrate.js +0 -104
- package/state/node-cost.js +0 -67
- package/state/node-executions.js +0 -775
- package/state/node-health-store.js +0 -124
- package/state/ops-merge.js +0 -93
- package/state/paths.js +0 -146
- package/state/pipeline-dirs.js +0 -99
- package/state/pipeline-registry.js +0 -449
- package/state/run-store.js +0 -697
- package/state/schema.js +0 -438
- package/state/schema.ops.js +0 -44
- package/state/session-index.js +0 -177
- package/state/session-keys.js +0 -39
- package/state/state-store.js +0 -311
- package/state/subflow-state.js +0 -126
- package/state/tool-handoff.js +0 -169
- package/telemetry/budget-alert.js +0 -353
- package/telemetry/cli.js +0 -331
- package/telemetry/grammar.js +0 -119
- package/telemetry/http-budget.js +0 -132
- package/telemetry/http-cost.js +0 -670
- package/telemetry/http-helpers.js +0 -134
- package/telemetry/http-tree.js +0 -349
- package/telemetry/http.js +0 -44
- package/telemetry/index.js +0 -307
- package/telemetry/ingest.js +0 -134
- package/telemetry/labels.js +0 -115
- package/telemetry/pricer.js +0 -77
- package/telemetry/resolver.js +0 -314
- package/telemetry/store.js +0 -147
- package/utils/run-patch.js +0 -43
|
@@ -1,124 +0,0 @@
|
|
|
1
|
-
// node-health-store.js — circuit breaker for any node's agent dispatch.
|
|
2
|
-
//
|
|
3
|
-
// Storage: the store home's runtime.sqlite (sibling of flow_branches, failure_events).
|
|
4
|
-
//
|
|
5
|
-
// Any "agent"-kind node can opt in via node.circuitBreaker: true, checked by
|
|
6
|
-
// the ordinary agent dispatch path regardless of which kind of flow it's
|
|
7
|
-
// running inside. When the agent dispatch fails, we record it here so any
|
|
8
|
-
// subsequent flow that reaches the same (pipelineId, nodeId) fails
|
|
9
|
-
// immediately instead of paying for another doomed LLM call — until an
|
|
10
|
-
// operator resets it.
|
|
11
|
-
//
|
|
12
|
-
// Column shape mirrors flow_runs (state/run-store.js / registry.sqlite): one
|
|
13
|
-
// `status` column drives state instead of nullable timestamp columns doing
|
|
14
|
-
// double duty as flags, plus generic created_at/updated_at/ended_at. A given
|
|
15
|
-
// (pipeline_id, node_id) can have many historical rows across separate block
|
|
16
|
-
// episodes; only one may be status='blocked' at a time, enforced by a partial
|
|
17
|
-
// unique index rather than a primary key on (pipeline_id, node_id) — resets
|
|
18
|
-
// flag the row resolved rather than deleting it, so history survives.
|
|
19
|
-
|
|
20
|
-
import { randomUUID } from "node:crypto";
|
|
21
|
-
import { eq, and } from "drizzle-orm";
|
|
22
|
-
import { getSharedDb, withSharedWrite, resolveStorePaths } from "./db.js";
|
|
23
|
-
import { enqueue, enqueueRaw, flush as flushBatchWriter } from "./batch-writer.js";
|
|
24
|
-
import { nodeHealth } from "./schema.js";
|
|
25
|
-
import path from "node:path";
|
|
26
|
-
import os from "node:os";
|
|
27
|
-
import { mkdirSync } from "node:fs";
|
|
28
|
-
import { migrateDatabase, RUNTIME_MIGRATIONS_FOLDER } from "./migrate.js";
|
|
29
|
-
|
|
30
|
-
function defaultDbPath() {
|
|
31
|
-
return resolveStorePaths().runtimeDb;
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
const INSERT_BLOCKED_SQL = `
|
|
35
|
-
INSERT OR IGNORE INTO node_health
|
|
36
|
-
(id, pipeline_id, node_id, status, reason, blocked_by_flow_id, created_at, updated_at)
|
|
37
|
-
VALUES (?, ?, ?, 'blocked', ?, ?, ?, ?)
|
|
38
|
-
`;
|
|
39
|
-
const RESOLVE_SQL = `
|
|
40
|
-
UPDATE node_health
|
|
41
|
-
SET status = 'resolved', resolved_via = ?, updated_at = ?, ended_at = ?
|
|
42
|
-
WHERE pipeline_id = ? AND node_id = ? AND status = 'blocked'
|
|
43
|
-
`;
|
|
44
|
-
const IS_BLOCKED_SQL = `
|
|
45
|
-
SELECT 1 FROM node_health WHERE pipeline_id = ? AND node_id = ? AND status = 'blocked' LIMIT 1
|
|
46
|
-
`;
|
|
47
|
-
|
|
48
|
-
export async function createNodeHealthStore({ logger, dbPath = defaultDbPath() } = {}) {
|
|
49
|
-
mkdirSync(path.dirname(dbPath), { recursive: true });
|
|
50
|
-
await migrateDatabase({ dbPath, migrationsFolder: RUNTIME_MIGRATIONS_FOLDER });
|
|
51
|
-
const { db, client } = await getSharedDb(dbPath);
|
|
52
|
-
|
|
53
|
-
async function isBlocked(pipelineId, nodeId) {
|
|
54
|
-
if (!pipelineId || !nodeId) return false;
|
|
55
|
-
const rows = await db.select({ x: nodeHealth.id }).from(nodeHealth)
|
|
56
|
-
.where(and(eq(nodeHealth.pipelineId, pipelineId), eq(nodeHealth.nodeId, nodeId), eq(nodeHealth.status, "blocked")))
|
|
57
|
-
.limit(1);
|
|
58
|
-
return rows.length > 0;
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
// First failure opens a new blocked row. A failure while already blocked
|
|
62
|
-
// is a silent no-op — the partial unique index rejects a second active
|
|
63
|
-
// row for the same (pipelineId, nodeId), so INSERT OR IGNORE just drops it.
|
|
64
|
-
//
|
|
65
|
-
// Deferred via batch-writer: its only caller (executor-nodes.js) awaits
|
|
66
|
-
// this purely to sequence before re-throwing the original dispatch error
|
|
67
|
-
// and never inspects a return value. rowsAffected (and therefore whether
|
|
68
|
-
// this was a genuinely new block vs. a no-op against an already-blocked
|
|
69
|
-
// row) isn't known until flush, so the old conditional "marked unhealthy"
|
|
70
|
-
// log is dropped rather than logged on a guess. reset() below stays
|
|
71
|
-
// immediate — its CLI caller reports a different message to the operator
|
|
72
|
-
// based on the real rowsAffected, so it can't be deferred the same way.
|
|
73
|
-
function markUnhealthy(pipelineId, nodeId, { reason, flowId } = {}) {
|
|
74
|
-
if (!pipelineId || !nodeId) return;
|
|
75
|
-
const now = Date.now();
|
|
76
|
-
// Raw on both statements: the batch-writer flushes drizzle and raw items
|
|
77
|
-
// as separate round-trips, and the registry row must land first.
|
|
78
|
-
enqueueRaw(dbPath, "INSERT OR IGNORE INTO pipeline_nodes (pipeline_id, node_name, first_seen_at) VALUES (?, ?, ?)", [pipelineId, nodeId, now]);
|
|
79
|
-
enqueueRaw(dbPath, `INSERT OR IGNORE INTO node_health
|
|
80
|
-
(id, pipeline_id, node_id, status, reason, blocked_by_flow_id, created_at, updated_at)
|
|
81
|
-
VALUES (?, ?, ?, 'blocked', ?, ?, ?, ?)`,
|
|
82
|
-
[randomUUID(), pipelineId, nodeId, reason ?? "unknown failure", flowId ?? null, now, now]);
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
async function reset(pipelineId, nodeId, { via } = {}) {
|
|
86
|
-
if (!pipelineId || !nodeId) return { reset: 0 };
|
|
87
|
-
const now = Date.now();
|
|
88
|
-
const r = await withSharedWrite(dbPath, () =>
|
|
89
|
-
db.update(nodeHealth)
|
|
90
|
-
.set({ status: "resolved", resolvedVia: via ?? "operator", updatedAt: now, endedAt: now })
|
|
91
|
-
.where(and(eq(nodeHealth.pipelineId, pipelineId), eq(nodeHealth.nodeId, nodeId), eq(nodeHealth.status, "blocked")))
|
|
92
|
-
);
|
|
93
|
-
if (r.rowsAffected > 0) {
|
|
94
|
-
logger?.info?.(`manifold node-health: ${pipelineId}/${nodeId} reset (${via ?? "operator"})`);
|
|
95
|
-
}
|
|
96
|
-
return { reset: r.rowsAffected };
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
async function list(pipelineId, { status } = {}) {
|
|
100
|
-
const where = [];
|
|
101
|
-
const args = [];
|
|
102
|
-
if (pipelineId) { where.push("pipeline_id = ?"); args.push(pipelineId); }
|
|
103
|
-
if (status) { where.push("status = ?"); args.push(status); }
|
|
104
|
-
const sql = `
|
|
105
|
-
SELECT * FROM node_health
|
|
106
|
-
${where.length ? `WHERE ${where.join(" AND ")}` : ""}
|
|
107
|
-
ORDER BY created_at DESC
|
|
108
|
-
`;
|
|
109
|
-
const { rows } = await client.execute({ sql, args });
|
|
110
|
-
return rows;
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
function close() {
|
|
114
|
-
client.close();
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
// markUnhealthy() defers through batch-writer — for tests that assert
|
|
118
|
-
// read-after-write. Not meant for production use.
|
|
119
|
-
async function flush() {
|
|
120
|
-
await flushBatchWriter(dbPath);
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
return { isBlocked, markUnhealthy, reset, list, flush, close, _db: client };
|
|
124
|
-
}
|
package/state/ops-merge.js
DELETED
|
@@ -1,93 +0,0 @@
|
|
|
1
|
-
// ops-merge.js — one-time copy of the lineage/cost tables out of ops.sqlite.
|
|
2
|
-
//
|
|
3
|
-
// Those tables now live in runtime.sqlite because flow_id and pipeline_id
|
|
4
|
-
// need FKs, and SQLite cannot constrain a column against a table in another
|
|
5
|
-
// file. The copy runs inside migrateDatabase() after the runtime snapshot,
|
|
6
|
-
// and is idempotent: rows already in runtime win (INSERT OR IGNORE), and a
|
|
7
|
-
// non-empty runtime table means the merge already happened.
|
|
8
|
-
//
|
|
9
|
-
// Columns are intersected rather than assumed equal: an ops file can lack a
|
|
10
|
-
// column its baseline migration declares (queue_items.kind), because CREATE
|
|
11
|
-
// TABLE IF NOT EXISTS never adds columns to a table that already exists. A
|
|
12
|
-
// missing column merges as its default/NULL instead of failing the boot.
|
|
13
|
-
|
|
14
|
-
import path from "node:path";
|
|
15
|
-
import { existsSync } from "node:fs";
|
|
16
|
-
import Database from "better-sqlite3";
|
|
17
|
-
|
|
18
|
-
const TABLES = {
|
|
19
|
-
queue_items: { flow: "flow_id", pipeline: "pipeline_id" },
|
|
20
|
-
session_lineage: { flow: "flow_id", pipeline: "pipeline_id" },
|
|
21
|
-
cost_by_call: { flow: "flow_id", pipeline: "pipeline_id" },
|
|
22
|
-
};
|
|
23
|
-
|
|
24
|
-
function columnsOf(db, schema, table) {
|
|
25
|
-
// prepare().columns() is schema-accurate; `SELECT name FROM ops.pragma_table_info(?)`
|
|
26
|
-
// silently reports the main-schema table of the same name.
|
|
27
|
-
return db.prepare(`SELECT * FROM "${schema}"."${table}" LIMIT 0`).columns().map((c) => c.name);
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
export function mergeOpsTables({ runtimeDbPath, logger = null } = {}) {
|
|
31
|
-
// Only the canonical pair is migrated; alternate paths (tests, one-off
|
|
32
|
-
// stores) would otherwise look for an unrelated sibling ops.sqlite.
|
|
33
|
-
if (path.basename(runtimeDbPath) !== "runtime.sqlite") {
|
|
34
|
-
return { merged: false, reason: "not-canonical" };
|
|
35
|
-
}
|
|
36
|
-
const opsPath = path.join(path.dirname(runtimeDbPath), "ops.sqlite");
|
|
37
|
-
if (!existsSync(opsPath)) return { merged: false, reason: "no-ops-db" };
|
|
38
|
-
|
|
39
|
-
const db = new Database(runtimeDbPath);
|
|
40
|
-
try {
|
|
41
|
-
db.exec("PRAGMA busy_timeout = 30000;");
|
|
42
|
-
const existing = Object.keys(TABLES).reduce(
|
|
43
|
-
(n, t) => n + db.prepare(`SELECT COUNT(*) AS n FROM main.${t}`).get().n, 0,
|
|
44
|
-
);
|
|
45
|
-
if (existing > 0) return { merged: false, reason: "already-populated" };
|
|
46
|
-
|
|
47
|
-
db.prepare("ATTACH DATABASE ? AS ops").run(opsPath);
|
|
48
|
-
const present = Object.keys(TABLES).filter(
|
|
49
|
-
(t) => db.prepare("SELECT 1 FROM ops.sqlite_master WHERE type='table' AND name=?").get(t),
|
|
50
|
-
);
|
|
51
|
-
if (present.length === 0) return { merged: false, reason: "no-source-tables" };
|
|
52
|
-
|
|
53
|
-
db.exec("PRAGMA foreign_keys = OFF;");
|
|
54
|
-
db.exec("BEGIN");
|
|
55
|
-
const copied = {};
|
|
56
|
-
try {
|
|
57
|
-
for (const table of present) {
|
|
58
|
-
const runtimeCols = columnsOf(db, "main", table);
|
|
59
|
-
const opsCols = new Set(columnsOf(db, "ops", table));
|
|
60
|
-
const cols = runtimeCols.filter((c) => opsCols.has(c));
|
|
61
|
-
const list = cols.map((c) => `"${c}"`).join(", ");
|
|
62
|
-
db.exec(`INSERT OR IGNORE INTO main."${table}" (${list}) SELECT ${list} FROM ops."${table}"`);
|
|
63
|
-
copied[table] = db.prepare(`SELECT COUNT(*) AS n FROM main."${table}"`).get().n;
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
// Historical rows whose run/pipeline was pruned predate the constraints.
|
|
67
|
-
// Null the pointer, keep the telemetry row; the FK is for writes.
|
|
68
|
-
for (const [table, cols] of Object.entries(TABLES)) {
|
|
69
|
-
if (!present.includes(table)) continue;
|
|
70
|
-
for (const [kind, col] of Object.entries(cols)) {
|
|
71
|
-
const parent = kind === "flow" ? "flow_runs" : "pipelines";
|
|
72
|
-
const parentCol = kind === "flow" ? "flow_id" : "id";
|
|
73
|
-
db.exec(`UPDATE main."${table}" SET "${col}" = NULL
|
|
74
|
-
WHERE "${col}" IS NOT NULL
|
|
75
|
-
AND "${col}" NOT IN (SELECT "${parentCol}" FROM main."${parent}")`);
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
db.exec("COMMIT");
|
|
79
|
-
} catch (err) {
|
|
80
|
-
db.exec("ROLLBACK");
|
|
81
|
-
throw err;
|
|
82
|
-
}
|
|
83
|
-
db.exec("PRAGMA foreign_keys = ON;");
|
|
84
|
-
|
|
85
|
-
logger?.info?.(`ops-merge: copied ${JSON.stringify(copied)} from ${opsPath}`);
|
|
86
|
-
return { merged: true, copied };
|
|
87
|
-
} catch (err) {
|
|
88
|
-
logger?.warn?.(`ops-merge: skipped (${err?.message ?? err})`);
|
|
89
|
-
return { merged: false, reason: "error" };
|
|
90
|
-
} finally {
|
|
91
|
-
db.close();
|
|
92
|
-
}
|
|
93
|
-
}
|
package/state/paths.js
DELETED
|
@@ -1,146 +0,0 @@
|
|
|
1
|
-
import os from "node:os";
|
|
2
|
-
import path from "node:path";
|
|
3
|
-
import { createRequire } from "node:module";
|
|
4
|
-
|
|
5
|
-
const require = createRequire(import.meta.url);
|
|
6
|
-
// Lazy CJS binding (not a static named import) so ESM test mocks of
|
|
7
|
-
// node:fs that only stub the functions they intercept keep linking.
|
|
8
|
-
const fs = require("node:fs");
|
|
9
|
-
const { copyFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } = fs;
|
|
10
|
-
|
|
11
|
-
export function defaultDataDir(home = os.homedir()) {
|
|
12
|
-
return path.join(home, ".manifold");
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
export function resolveDataDir({ env = process.env, home = os.homedir() } = {}) {
|
|
16
|
-
return env.MANIFOLD_DATA_DIR || defaultDataDir(home);
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
export function runtimeDbPath(dataDir) {
|
|
20
|
-
return path.join(dataDir, "runtime.sqlite");
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
export function opsDbPath(dataDir) {
|
|
24
|
-
return path.join(dataDir, "ops.sqlite");
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
export function pipelinesDir(dataDir) {
|
|
28
|
-
return path.join(dataDir, "workspace", ".pipelines");
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
export function mediaAudioDir(dataDir) {
|
|
32
|
-
return path.join(dataDir, "media", "audio");
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
export function harnessMediaAudioDir(dataDir, harness) {
|
|
36
|
-
const safe = String(harness ?? "").replace(/[^A-Za-z0-9._-]+/g, "_");
|
|
37
|
-
if (!safe) throw new Error("manifold: harness is required to resolve the media dir (declare it on the connection)");
|
|
38
|
-
return path.join(mediaAudioDir(dataDir), safe);
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
export function stateFile(dataDir) {
|
|
42
|
-
return path.join(dataDir, "state", "manifold-paused.json");
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
function migrationMarkerPath(dataDir) {
|
|
46
|
-
return path.join(dataDir, ".migrated.json");
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
function legacyDbPath(home, name) {
|
|
50
|
-
return path.join(home, ".openclaw", name);
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
export function ensureMigrated({ dataDir, home = os.homedir(), logger = null } = {}) {
|
|
54
|
-
try {
|
|
55
|
-
mkdirSync(dataDir, { recursive: true });
|
|
56
|
-
} catch (err) {
|
|
57
|
-
logger?.warn?.(`paths: cannot create data dir ${dataDir}: ${err?.message ?? err}`);
|
|
58
|
-
return { migrated: false };
|
|
59
|
-
}
|
|
60
|
-
if (existsSync(migrationMarkerPath(dataDir))) return { migrated: false };
|
|
61
|
-
const runtime = runtimeDbPath(dataDir);
|
|
62
|
-
const ops = opsDbPath(dataDir);
|
|
63
|
-
if (existsSync(runtime) && existsSync(ops)) {
|
|
64
|
-
try {
|
|
65
|
-
writeFileSync(migrationMarkerPath(dataDir), JSON.stringify({ at: new Date().toISOString(), from: null }) + "\n");
|
|
66
|
-
} catch (err) {
|
|
67
|
-
logger?.warn?.(`paths: cannot write migration marker: ${err?.message ?? err}`);
|
|
68
|
-
}
|
|
69
|
-
return { migrated: false };
|
|
70
|
-
}
|
|
71
|
-
const legacyRuntime = legacyDbPath(home, "runtime.sqlite");
|
|
72
|
-
const legacyOps = legacyDbPath(home, "ops.sqlite");
|
|
73
|
-
const sources = [];
|
|
74
|
-
if (existsSync(legacyRuntime) && !existsSync(runtime)) sources.push([legacyRuntime, runtime]);
|
|
75
|
-
if (existsSync(legacyOps) && !existsSync(ops)) sources.push([legacyOps, ops]);
|
|
76
|
-
if (sources.length === 0) {
|
|
77
|
-
try {
|
|
78
|
-
writeFileSync(migrationMarkerPath(dataDir), JSON.stringify({ at: new Date().toISOString(), from: null, fresh: true }) + "\n");
|
|
79
|
-
} catch (err) {
|
|
80
|
-
logger?.warn?.(`paths: cannot write migration marker: ${err?.message ?? err}`);
|
|
81
|
-
}
|
|
82
|
-
return { migrated: false };
|
|
83
|
-
}
|
|
84
|
-
for (const [src, dest] of sources) {
|
|
85
|
-
copyFileSync(src, dest);
|
|
86
|
-
logger?.info?.(`paths: migrated ${src} -> ${dest}`);
|
|
87
|
-
}
|
|
88
|
-
let verified = true;
|
|
89
|
-
try {
|
|
90
|
-
verifyCopies(sources.map(([, dest]) => dest));
|
|
91
|
-
} catch (err) {
|
|
92
|
-
if (err?.code !== "MANIFOLD_VERIFY_UNAVAILABLE") throw err;
|
|
93
|
-
verified = false;
|
|
94
|
-
logger?.error?.(`paths: migrated copies UNVERIFIED: ${err?.message ?? err}`);
|
|
95
|
-
}
|
|
96
|
-
try {
|
|
97
|
-
writeFileSync(
|
|
98
|
-
migrationMarkerPath(dataDir),
|
|
99
|
-
JSON.stringify({ at: new Date().toISOString(), from: sources.map(([src]) => src), verified }) + "\n",
|
|
100
|
-
);
|
|
101
|
-
} catch (err) {
|
|
102
|
-
logger?.warn?.(`paths: cannot write migration marker: ${err?.message ?? err}`);
|
|
103
|
-
}
|
|
104
|
-
return { migrated: true, from: sources.map(([src]) => src) };
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
function verifyCopies(destPaths) {
|
|
108
|
-
for (const dest of destPaths) {
|
|
109
|
-
let Database;
|
|
110
|
-
try {
|
|
111
|
-
const mod = require("better-sqlite3");
|
|
112
|
-
Database = mod?.default ?? mod;
|
|
113
|
-
} catch {
|
|
114
|
-
const err = new Error(`paths: cannot verify migrated copy ${dest} (better-sqlite3 unavailable)`);
|
|
115
|
-
err.code = "MANIFOLD_VERIFY_UNAVAILABLE";
|
|
116
|
-
throw err;
|
|
117
|
-
}
|
|
118
|
-
const db = new Database(dest, { readonly: true });
|
|
119
|
-
try {
|
|
120
|
-
const row = db.prepare("PRAGMA quick_check;").get();
|
|
121
|
-
const out = row ? Object.values(row)[0] : null;
|
|
122
|
-
if (out !== "ok") throw new Error(`paths: migrated copy failed quick_check: ${dest}: ${out}`);
|
|
123
|
-
} finally {
|
|
124
|
-
db.close();
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
export function readMigrationMarker(dataDir) {
|
|
130
|
-
try {
|
|
131
|
-
return JSON.parse(readFileSync(migrationMarkerPath(dataDir), "utf8"));
|
|
132
|
-
} catch {
|
|
133
|
-
return null;
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
export function resolveStorePaths({ paths = null, dataDir = null, env = process.env } = {}) {
|
|
138
|
-
const dir = paths?.dataDir ?? dataDir ?? resolveDataDir({ env });
|
|
139
|
-
return {
|
|
140
|
-
dataDir: dir,
|
|
141
|
-
runtimeDb: paths?.runtimeDb ?? env.MANIFOLD_RUNTIME_DB ?? runtimeDbPath(dir),
|
|
142
|
-
opsDb: paths?.opsDb ?? env.MANIFOLD_OPS_DB ?? opsDbPath(dir),
|
|
143
|
-
pipelinesDir: paths?.pipelinesDir ?? env.MANIFOLD_PIPELINES_DIR ?? pipelinesDir(dir),
|
|
144
|
-
mediaAudioDir: paths?.mediaAudioDir ?? env.MANIFOLD_MEDIA_AUDIO_DIR ?? mediaAudioDir(dir),
|
|
145
|
-
};
|
|
146
|
-
}
|
package/state/pipeline-dirs.js
DELETED
|
@@ -1,99 +0,0 @@
|
|
|
1
|
-
// pipeline-dirs.js — per-harness pipeline directory overrides.
|
|
2
|
-
//
|
|
3
|
-
// Resolution order (resolvePipelinesDir): table[harness] →
|
|
4
|
-
// MANIFOLD_PIPELINES_DIR env → adapter config `pipelinesDir` →
|
|
5
|
-
// <dataDir>/workspace/.pipelines default. Table wins so an operator can
|
|
6
|
-
// relocate one harness without touching env or config; env wins over config
|
|
7
|
-
// so a one-shot override doesn't need a config edit.
|
|
8
|
-
//
|
|
9
|
-
// Sync better-sqlite3 open-per-operation (same posture as migrate.js's own
|
|
10
|
-
// open/close): no persistent handle, so the gateway and one-shot CLI
|
|
11
|
-
// processes never share lock state — WAL + busy_timeout carry the
|
|
12
|
-
// cross-process safety. Every op runs pending migrations first, so a fresh
|
|
13
|
-
// file self-creates the table.
|
|
14
|
-
|
|
15
|
-
import path from "node:path";
|
|
16
|
-
import { existsSync, mkdirSync, statSync } from "node:fs";
|
|
17
|
-
import Database from "better-sqlite3";
|
|
18
|
-
import { resolveDataDir, pipelinesDir as defaultPipelinesDir } from "./paths.js";
|
|
19
|
-
import { migrateDatabase, RUNTIME_MIGRATIONS_FOLDER } from "./migrate.js";
|
|
20
|
-
|
|
21
|
-
export function createPipelineDirsStore({ dbPath, logger = null } = {}) {
|
|
22
|
-
function withDb(fn) {
|
|
23
|
-
mkdirSync(path.dirname(dbPath), { recursive: true });
|
|
24
|
-
migrateDatabase({ dbPath, migrationsFolder: RUNTIME_MIGRATIONS_FOLDER, logger });
|
|
25
|
-
const db = new Database(dbPath);
|
|
26
|
-
try {
|
|
27
|
-
db.exec("PRAGMA busy_timeout = 30000;");
|
|
28
|
-
return fn(db);
|
|
29
|
-
} finally {
|
|
30
|
-
db.close();
|
|
31
|
-
}
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
// Upsert. Throws when the directory does not exist — a registered path
|
|
35
|
-
// that isn't there is always an operator typo, never a valid override.
|
|
36
|
-
function register({ harness, dirPath } = {}) {
|
|
37
|
-
const harnessName = String(harness ?? "").trim();
|
|
38
|
-
if (!harnessName) throw new Error("pipeline-dirs: register needs a non-empty harness");
|
|
39
|
-
const resolved = path.resolve(String(dirPath ?? ""));
|
|
40
|
-
let st = null;
|
|
41
|
-
try { st = statSync(resolved); } catch { st = null; }
|
|
42
|
-
if (!st?.isDirectory()) throw new Error(`pipeline-dirs: not a directory: ${resolved}`);
|
|
43
|
-
const now = Date.now();
|
|
44
|
-
withDb((db) => {
|
|
45
|
-
// pipeline_dirs.harness carries an FK; an operator registering the
|
|
46
|
-
// first override for a harness must not hit a constraint error.
|
|
47
|
-
db.transaction(() => {
|
|
48
|
-
db.prepare(
|
|
49
|
-
`INSERT INTO harnesses (id, created_at, updated_at) VALUES (?, ?, ?)
|
|
50
|
-
ON CONFLICT(id) DO UPDATE SET updated_at = excluded.updated_at`
|
|
51
|
-
).run(harnessName, now, now);
|
|
52
|
-
db.prepare(
|
|
53
|
-
`INSERT INTO pipeline_dirs (harness, path, created_at) VALUES (?, ?, ?)
|
|
54
|
-
ON CONFLICT(harness) DO UPDATE SET path = excluded.path`
|
|
55
|
-
).run(harnessName, resolved, now);
|
|
56
|
-
})();
|
|
57
|
-
});
|
|
58
|
-
return { harness: harnessName, path: resolved };
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
function list() {
|
|
62
|
-
try {
|
|
63
|
-
return withDb((db) =>
|
|
64
|
-
db.prepare("SELECT harness, path, created_at FROM pipeline_dirs ORDER BY harness").all()
|
|
65
|
-
);
|
|
66
|
-
} catch (err) {
|
|
67
|
-
logger?.warn?.(`pipeline-dirs: list failed: ${err?.message ?? err}`);
|
|
68
|
-
return [];
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
function get(harness) {
|
|
73
|
-
if (!harness) return null;
|
|
74
|
-
try {
|
|
75
|
-
return withDb((db) =>
|
|
76
|
-
db.prepare("SELECT harness, path, created_at FROM pipeline_dirs WHERE harness = ?").get(String(harness)) ?? null
|
|
77
|
-
);
|
|
78
|
-
} catch (err) {
|
|
79
|
-
logger?.warn?.(`pipeline-dirs: get failed: ${err?.message ?? err}`);
|
|
80
|
-
return null;
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
// Pure precedence over an explicit lookup + env + config + default — the
|
|
85
|
-
// store method below delegates here so tests pin the order without a DB.
|
|
86
|
-
function resolve({ harness = null, env = process.env, dataDir = null, configDir = null, lookup = null } = {}) {
|
|
87
|
-
const fromTable = typeof lookup === "function" ? lookup(harness) : null;
|
|
88
|
-
if (fromTable?.path) return { dir: fromTable.path, source: "table" };
|
|
89
|
-
if (env?.MANIFOLD_PIPELINES_DIR) return { dir: env.MANIFOLD_PIPELINES_DIR, source: "env" };
|
|
90
|
-
if (configDir) return { dir: configDir, source: "config" };
|
|
91
|
-
return { dir: defaultPipelinesDir(dataDir ?? resolveDataDir({ env })), source: "default" };
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
function resolveDir({ harness = null, env = process.env, dataDir = null, configDir = null } = {}) {
|
|
95
|
-
return resolve({ harness, env, dataDir, configDir, lookup: (h) => get(h) });
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
return { register, list, get, resolve, resolveDir, close() {} };
|
|
99
|
-
}
|