@hasna/loops 0.4.26 → 0.4.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +121 -0
- package/README.md +6 -0
- package/dist/api/index.js +689 -20
- package/dist/cli/index.js +509 -85
- package/dist/daemon/index.js +118 -11
- package/dist/index.js +525 -64
- package/dist/lib/executor.d.ts +9 -0
- package/dist/lib/migration.d.ts +66 -0
- package/dist/lib/mode.js +3 -3
- package/dist/lib/route/fields.d.ts +8 -0
- package/dist/lib/storage/index.js +197 -7
- package/dist/lib/storage/postgres-loop-storage.d.ts +2 -2
- package/dist/lib/storage/postgres-schema.js +3 -0
- package/dist/lib/storage/postgres.js +3 -0
- package/dist/lib/storage/sqlite.js +83 -3
- package/dist/lib/store/index.d.ts +3 -0
- package/dist/lib/store.d.ts +77 -0
- package/dist/lib/store.js +89 -4
- package/dist/mcp/index.js +118 -11
- package/dist/runner/index.js +35 -8
- package/dist/sdk/http.d.ts +153 -1
- package/dist/sdk/http.js +78 -1
- package/dist/sdk/index.js +411 -60
- package/dist/serve/index.js +919 -60
- package/dist/types.d.ts +11 -0
- package/docs/DEPLOYMENT_MODES.md +6 -0
- package/docs/USAGE.md +11 -8
- package/package.json +3 -3
package/dist/cli/index.js
CHANGED
|
@@ -1571,6 +1571,7 @@ var DEFAULT_RECOVERY_BATCH_LIMIT = 100;
|
|
|
1571
1571
|
var DEFAULT_RECOVERY_SCAN_MULTIPLIER = 5;
|
|
1572
1572
|
var LIVE_EXPIRED_RUN_GRACE_MS = 60000;
|
|
1573
1573
|
var SCHEMA_USER_VERSION = 8;
|
|
1574
|
+
var BREAKING_SCHEMA_FLOOR = 7;
|
|
1574
1575
|
var TERMINAL_RUN_STATUSES = ["succeeded", "failed", "timed_out", "abandoned", "skipped"];
|
|
1575
1576
|
var PRUNE_BATCH_SIZE = 400;
|
|
1576
1577
|
var GENERATED_ROUTE_TEMPLATE_IDS = new Set(["todos-task-worker-verifier", "task-lifecycle", "event-worker-verifier"]);
|
|
@@ -1713,6 +1714,7 @@ function rowToWorkflowWorkItem(row) {
|
|
|
1713
1714
|
priority: row.priority,
|
|
1714
1715
|
status: row.status,
|
|
1715
1716
|
attempts: row.attempts,
|
|
1717
|
+
gateDeaths: row.gate_deaths ?? 0,
|
|
1716
1718
|
nextAttemptAt: row.next_attempt_at ?? undefined,
|
|
1717
1719
|
leaseExpiresAt: row.lease_expires_at ?? undefined,
|
|
1718
1720
|
workflowId: row.workflow_id ?? undefined,
|
|
@@ -1862,6 +1864,23 @@ function workItemStatusForLoopRun(status, attempt, maxAttempts) {
|
|
|
1862
1864
|
}
|
|
1863
1865
|
return;
|
|
1864
1866
|
}
|
|
1867
|
+
var WORK_ITEM_TEMPFAIL_EXIT_CODE = 75;
|
|
1868
|
+
var GATE_STEP_IDS = new Set(["triage", "planner", "plan"]);
|
|
1869
|
+
var GATE_DEATH_MAX_DURATION_MS = 60000;
|
|
1870
|
+
var GATE_DEATH_CEILING = 20;
|
|
1871
|
+
function classifyNonProductiveStepFailure(steps) {
|
|
1872
|
+
const failing = [...steps].reverse().find((step) => step.status === "failed" || step.status === "timed_out");
|
|
1873
|
+
if (!failing)
|
|
1874
|
+
return;
|
|
1875
|
+
if (failing.exitCode === WORK_ITEM_TEMPFAIL_EXIT_CODE)
|
|
1876
|
+
return "tempfail";
|
|
1877
|
+
if (typeof failing.error === "string" && failing.error.includes("worktree preparation failed"))
|
|
1878
|
+
return "gate-death";
|
|
1879
|
+
const fast = failing.durationMs === undefined || failing.durationMs < GATE_DEATH_MAX_DURATION_MS;
|
|
1880
|
+
if (GATE_STEP_IDS.has(failing.stepId) && fast)
|
|
1881
|
+
return "gate-death";
|
|
1882
|
+
return;
|
|
1883
|
+
}
|
|
1865
1884
|
function scrubbedOrNull(value) {
|
|
1866
1885
|
return value == null ? null : scrubSecrets(value);
|
|
1867
1886
|
}
|
|
@@ -1962,11 +1981,21 @@ class Store {
|
|
|
1962
1981
|
id TEXT PRIMARY KEY,
|
|
1963
1982
|
applied_at TEXT NOT NULL
|
|
1964
1983
|
);
|
|
1984
|
+
CREATE TABLE IF NOT EXISTS schema_compat (
|
|
1985
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
1986
|
+
min_compatible_user_version INTEGER NOT NULL
|
|
1987
|
+
);
|
|
1965
1988
|
`);
|
|
1966
1989
|
const versionRow = this.db.query("PRAGMA user_version").get();
|
|
1967
1990
|
const userVersion = versionRow?.user_version ?? 0;
|
|
1968
1991
|
if (userVersion > SCHEMA_USER_VERSION) {
|
|
1969
|
-
|
|
1992
|
+
const floorRow = this.db.query("SELECT min_compatible_user_version FROM schema_compat WHERE id = 1").get();
|
|
1993
|
+
if (!floorRow) {
|
|
1994
|
+
throw new Error(`loops database schema version ${userVersion} is newer than this binary supports (${SCHEMA_USER_VERSION}) and carries no compatibility floor; upgrade open-loops before opening this database`);
|
|
1995
|
+
}
|
|
1996
|
+
if (SCHEMA_USER_VERSION < floorRow.min_compatible_user_version) {
|
|
1997
|
+
throw new Error(`loops database schema version ${userVersion} requires a binary with schema support >= ${floorRow.min_compatible_user_version} (this binary supports ${SCHEMA_USER_VERSION}); upgrade open-loops before opening this database`);
|
|
1998
|
+
}
|
|
1970
1999
|
}
|
|
1971
2000
|
const applied = new Set(this.db.query("SELECT id FROM schema_migrations").all().map((row) => row.id));
|
|
1972
2001
|
for (const migration of this.migrations()) {
|
|
@@ -1977,8 +2006,10 @@ class Store {
|
|
|
1977
2006
|
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run(migration.id, nowIso());
|
|
1978
2007
|
}
|
|
1979
2008
|
}
|
|
1980
|
-
if (userVersion
|
|
2009
|
+
if (userVersion < SCHEMA_USER_VERSION)
|
|
1981
2010
|
this.db.exec(`PRAGMA user_version = ${SCHEMA_USER_VERSION}`);
|
|
2011
|
+
this.db.query(`INSERT INTO schema_compat (id, min_compatible_user_version) VALUES (1, ?)
|
|
2012
|
+
ON CONFLICT(id) DO UPDATE SET min_compatible_user_version = MAX(min_compatible_user_version, excluded.min_compatible_user_version)`).run(BREAKING_SCHEMA_FLOOR);
|
|
1982
2013
|
}
|
|
1983
2014
|
migrations() {
|
|
1984
2015
|
return [
|
|
@@ -2045,6 +2076,12 @@ class Store {
|
|
|
2045
2076
|
this.addColumnIfMissing("workflow_work_items", "machine_id", "TEXT");
|
|
2046
2077
|
this.db.exec("CREATE INDEX IF NOT EXISTS idx_workflow_work_items_machine ON workflow_work_items(machine_id, status)");
|
|
2047
2078
|
}
|
|
2079
|
+
},
|
|
2080
|
+
{
|
|
2081
|
+
id: "0011_work_item_gate_deaths",
|
|
2082
|
+
apply: () => {
|
|
2083
|
+
this.addColumnIfMissing("workflow_work_items", "gate_deaths", "INTEGER NOT NULL DEFAULT 0");
|
|
2084
|
+
}
|
|
2048
2085
|
}
|
|
2049
2086
|
];
|
|
2050
2087
|
}
|
|
@@ -3175,7 +3212,7 @@ class Store {
|
|
|
3175
3212
|
}
|
|
3176
3213
|
upsertWorkflowWorkItem(input) {
|
|
3177
3214
|
const now = nowIso();
|
|
3178
|
-
const id = genId();
|
|
3215
|
+
const id = input.id ?? genId();
|
|
3179
3216
|
const status = input.status ?? "queued";
|
|
3180
3217
|
this.db.query(`INSERT INTO workflow_work_items (id, route_key, idempotency_key, invocation_id, source_type, source_ref,
|
|
3181
3218
|
subject_ref, project_key, project_group, machine_id, route_scope, priority, status, attempts, next_attempt_at, lease_expires_at,
|
|
@@ -3303,6 +3340,7 @@ class Store {
|
|
|
3303
3340
|
const placeholders = requeueableStatuses.map(() => "?").join(",");
|
|
3304
3341
|
const res = this.db.query(`UPDATE workflow_work_items
|
|
3305
3342
|
SET status='queued', workflow_id=NULL, loop_id=NULL, workflow_run_id=NULL,
|
|
3343
|
+
${patch.resetAttempts ? "attempts=0, gate_deaths=0," : ""}
|
|
3306
3344
|
next_attempt_at=NULL, lease_expires_at=NULL, last_reason=?, updated_at=?
|
|
3307
3345
|
WHERE id=? AND status IN (${placeholders})`).run(reason, now, id, ...requeueableStatuses);
|
|
3308
3346
|
const item = this.getWorkflowWorkItem(id);
|
|
@@ -3312,6 +3350,46 @@ class Store {
|
|
|
3312
3350
|
throw new Error(`workflow work item was not requeued: ${id} status=${item.status}`);
|
|
3313
3351
|
return item;
|
|
3314
3352
|
}
|
|
3353
|
+
deadLetterWorkflowWorkItem(id, patch = {}) {
|
|
3354
|
+
const now = nowIso();
|
|
3355
|
+
const reason = patch.reason?.trim() || "redispatch cap reached; dead-lettered";
|
|
3356
|
+
this.db.query(`UPDATE workflow_work_items
|
|
3357
|
+
SET status='dead_letter', next_attempt_at=NULL, lease_expires_at=NULL, last_reason=?, updated_at=?
|
|
3358
|
+
WHERE id=? AND status IN ('succeeded','failed','cancelled')`).run(reason, now, id);
|
|
3359
|
+
const item = this.getWorkflowWorkItem(id);
|
|
3360
|
+
if (!item)
|
|
3361
|
+
throw new Error(`workflow work item not found after dead-letter: ${id}`);
|
|
3362
|
+
return item;
|
|
3363
|
+
}
|
|
3364
|
+
demoteNonProductiveWorkItems(workflowRunId, finishedAt) {
|
|
3365
|
+
const kind = classifyNonProductiveStepFailure(this.listWorkflowStepRuns(workflowRunId));
|
|
3366
|
+
if (!kind) {
|
|
3367
|
+
this.db.query("UPDATE workflow_work_items SET gate_deaths=0, updated_at=? WHERE workflow_run_id=? AND status='failed' AND gate_deaths > 0").run(finishedAt, workflowRunId);
|
|
3368
|
+
return;
|
|
3369
|
+
}
|
|
3370
|
+
if (kind === "tempfail") {
|
|
3371
|
+
this.db.query(`UPDATE workflow_work_items
|
|
3372
|
+
SET status='queued', attempts=CASE WHEN attempts > 0 THEN attempts - 1 ELSE 0 END,
|
|
3373
|
+
gate_deaths=0,
|
|
3374
|
+
workflow_id=NULL, loop_id=NULL, workflow_run_id=NULL,
|
|
3375
|
+
next_attempt_at=NULL, lease_expires_at=NULL,
|
|
3376
|
+
last_reason='worker exited 75 (tempfail): requeued for retry; attempt refunded (does not count toward redispatch cap)',
|
|
3377
|
+
updated_at=?
|
|
3378
|
+
WHERE workflow_run_id=? AND status='failed'`).run(finishedAt, workflowRunId);
|
|
3379
|
+
return;
|
|
3380
|
+
}
|
|
3381
|
+
this.db.query(`UPDATE workflow_work_items
|
|
3382
|
+
SET attempts=CASE WHEN attempts > 0 THEN attempts - 1 ELSE 0 END,
|
|
3383
|
+
gate_deaths=gate_deaths + 1,
|
|
3384
|
+
status=CASE WHEN gate_deaths + 1 >= ${GATE_DEATH_CEILING} THEN 'dead_letter' ELSE status END,
|
|
3385
|
+
last_reason=CASE
|
|
3386
|
+
WHEN gate_deaths + 1 >= ${GATE_DEATH_CEILING}
|
|
3387
|
+
THEN 'gate-death ceiling reached (' || (gate_deaths + 1) || '/${GATE_DEATH_CEILING} consecutive runs died at worktree prep / triage / planner without reaching the worker): dead-lettered \u2014 the infrastructure fault needs an operator; ''loops routes requeue'' resets and retries'
|
|
3388
|
+
ELSE 'gate death before real work (worktree prep / triage / planner): attempt refunded (does not count toward redispatch cap); consecutive gate deaths: ' || (gate_deaths + 1) || '/${GATE_DEATH_CEILING}'
|
|
3389
|
+
END,
|
|
3390
|
+
updated_at=?
|
|
3391
|
+
WHERE workflow_run_id=? AND status='failed'`).run(finishedAt, workflowRunId);
|
|
3392
|
+
}
|
|
3315
3393
|
admitWorkflowWorkItem(id, patch) {
|
|
3316
3394
|
const now = nowIso();
|
|
3317
3395
|
const res = this.db.query(`UPDATE workflow_work_items
|
|
@@ -4023,6 +4101,8 @@ class Store {
|
|
|
4023
4101
|
if (changed) {
|
|
4024
4102
|
const itemStatus = status === "succeeded" ? "succeeded" : status === "cancelled" ? "cancelled" : "failed";
|
|
4025
4103
|
this.setWorkflowWorkItemsForWorkflowRun(workflowRunId, itemStatus, error, finishedAt);
|
|
4104
|
+
if (itemStatus === "failed")
|
|
4105
|
+
this.demoteNonProductiveWorkItems(workflowRunId, finishedAt);
|
|
4026
4106
|
this.maybeArchiveTerminalGeneratedRouteWorkflow(workflowRunId, finishedAt);
|
|
4027
4107
|
}
|
|
4028
4108
|
this.db.exec("COMMIT");
|
|
@@ -4972,7 +5052,7 @@ class Store {
|
|
|
4972
5052
|
// package.json
|
|
4973
5053
|
var package_default = {
|
|
4974
5054
|
name: "@hasna/loops",
|
|
4975
|
-
version: "0.4.
|
|
5055
|
+
version: "0.4.28",
|
|
4976
5056
|
description: "Persistent local loop and workflow runner for deterministic commands and headless AI coding agents",
|
|
4977
5057
|
type: "module",
|
|
4978
5058
|
main: "dist/index.js",
|
|
@@ -5082,6 +5162,7 @@ var package_default = {
|
|
|
5082
5162
|
bun: ">=1.0.0"
|
|
5083
5163
|
},
|
|
5084
5164
|
dependencies: {
|
|
5165
|
+
"@hasna/contracts": "^0.5.1",
|
|
5085
5166
|
"@hasna/events": "^0.1.9",
|
|
5086
5167
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
5087
5168
|
"@openrouter/ai-sdk-provider": "2.9.1",
|
|
@@ -5091,8 +5172,7 @@ var package_default = {
|
|
|
5091
5172
|
zod: "4.4.3"
|
|
5092
5173
|
},
|
|
5093
5174
|
optionalDependencies: {
|
|
5094
|
-
"@hasna/machines": "0.0.49"
|
|
5095
|
-
"@hasna/contracts": "^0.4.2"
|
|
5175
|
+
"@hasna/machines": "0.0.49"
|
|
5096
5176
|
},
|
|
5097
5177
|
devDependencies: {
|
|
5098
5178
|
"@types/bun": "latest",
|
|
@@ -6802,11 +6882,30 @@ function remoteWorktreePrepareLines(worktree) {
|
|
|
6802
6882
|
' git -C "$repo" rev-parse --is-inside-work-tree >/dev/null 2>&1 || { echo "worktree repoRoot is not a git repository: $repo" >&2; return 1; }',
|
|
6803
6883
|
' mkdir -p "$(dirname "$path")" || return 1',
|
|
6804
6884
|
" # Preparation chatter goes to stderr so run stdout stays the agent's.",
|
|
6805
|
-
|
|
6806
|
-
' git -C "$repo"
|
|
6807
|
-
"
|
|
6808
|
-
|
|
6885
|
+
" __openloops_worktree_add() {",
|
|
6886
|
+
' if git -C "$repo" show-ref --verify --quiet "refs/heads/$branch"; then',
|
|
6887
|
+
' git -C "$repo" worktree add "$path" "$branch"',
|
|
6888
|
+
" else",
|
|
6889
|
+
' git -C "$repo" worktree add -b "$branch" "$path" HEAD',
|
|
6890
|
+
" fi",
|
|
6891
|
+
" }",
|
|
6892
|
+
" local __ol_add_out",
|
|
6893
|
+
' if __ol_add_out="$(__openloops_worktree_add 2>&1)"; then',
|
|
6894
|
+
' if [ -n "$__ol_add_out" ]; then printf "%s\\n" "$__ol_add_out" >&2; fi',
|
|
6895
|
+
" return 0",
|
|
6809
6896
|
" fi",
|
|
6897
|
+
' printf "%s\\n" "$__ol_add_out" >&2',
|
|
6898
|
+
" # Self-heal git's own remedy for a stale 'missing but already registered",
|
|
6899
|
+
" # worktree': prune the dead registration (metadata-only; the directory is",
|
|
6900
|
+
" # already gone) and retry the add exactly once, then fail honestly.",
|
|
6901
|
+
' case "$__ol_add_out" in',
|
|
6902
|
+
' *"missing but already registered worktree"*)',
|
|
6903
|
+
' git -C "$repo" worktree prune 1>&2 || true',
|
|
6904
|
+
" __openloops_worktree_add 1>&2 || return 1",
|
|
6905
|
+
" return 0",
|
|
6906
|
+
" ;;",
|
|
6907
|
+
" esac",
|
|
6908
|
+
" return 1",
|
|
6810
6909
|
"}"
|
|
6811
6910
|
];
|
|
6812
6911
|
}
|
|
@@ -6946,6 +7045,9 @@ async function preflightNativeAuthProfile(spec, env) {
|
|
|
6946
7045
|
function spawnDetail(result) {
|
|
6947
7046
|
return (result.stderr || result.stdout || result.error || "").toString().trim();
|
|
6948
7047
|
}
|
|
7048
|
+
function isStaleWorktreeRegistration(detail) {
|
|
7049
|
+
return typeof detail === "string" && /missing but already registered worktree/i.test(detail);
|
|
7050
|
+
}
|
|
6949
7051
|
function resolvedDirEquals(left, right) {
|
|
6950
7052
|
try {
|
|
6951
7053
|
return realpathSync(left) === realpathSync(right);
|
|
@@ -7047,7 +7149,12 @@ async function ensureLocalWorktree(worktree, env) {
|
|
|
7047
7149
|
return { error: `could not create worktree parent directory: ${error instanceof Error ? error.message : String(error)}` };
|
|
7048
7150
|
}
|
|
7049
7151
|
const hasBranch = await git(["-C", repoRoot, "show-ref", "--verify", "--quiet", `refs/heads/${branch}`]);
|
|
7050
|
-
const
|
|
7152
|
+
const runAdd = () => hasBranch.status === 0 ? git(["-C", repoRoot, "worktree", "add", path, branch]) : git(["-C", repoRoot, "worktree", "add", "-b", branch, path, "HEAD"]);
|
|
7153
|
+
let add = await runAdd();
|
|
7154
|
+
if ((add.error || (add.status ?? 1) !== 0) && isStaleWorktreeRegistration(spawnDetail(add))) {
|
|
7155
|
+
await git(["-C", repoRoot, "worktree", "prune"]);
|
|
7156
|
+
add = await runAdd();
|
|
7157
|
+
}
|
|
7051
7158
|
if (add.error || (add.status ?? 1) !== 0) {
|
|
7052
7159
|
const detail = spawnDetail(add);
|
|
7053
7160
|
return { error: `git worktree add failed for ${path}${detail ? `: ${detail}` : ""}` };
|
|
@@ -10634,6 +10741,7 @@ function timeOnly(value) {
|
|
|
10634
10741
|
import { createHash as createHash4 } from "crypto";
|
|
10635
10742
|
import { hostname as hostname4 } from "os";
|
|
10636
10743
|
var LOOPS_MIGRATION_SCHEMA = "open-loops.migration/v1";
|
|
10744
|
+
var LOOPS_SELF_HOSTED_PUSH_MANIFEST_SCHEMA = "open-loops.self-hosted-push-manifest/v1";
|
|
10637
10745
|
function canonicalize(value) {
|
|
10638
10746
|
if (Array.isArray(value))
|
|
10639
10747
|
return value.map((entry) => canonicalize(entry));
|
|
@@ -10794,6 +10902,20 @@ function compareResource(current, incoming, opts) {
|
|
|
10794
10902
|
currentHash
|
|
10795
10903
|
};
|
|
10796
10904
|
}
|
|
10905
|
+
function remoteRepresentationRow(current, incoming, opts) {
|
|
10906
|
+
const incomingHash = migrationHash(incoming);
|
|
10907
|
+
if (!current)
|
|
10908
|
+
return { resource: opts.resource, id: opts.id, name: opts.name, action: "insert", incomingHash };
|
|
10909
|
+
return {
|
|
10910
|
+
resource: opts.resource,
|
|
10911
|
+
id: opts.id,
|
|
10912
|
+
name: opts.name,
|
|
10913
|
+
action: "skip",
|
|
10914
|
+
reason: "remote id is represented; self-hosted list payload is public/redacted, so exact byte comparison is reserved for import apply",
|
|
10915
|
+
incomingHash,
|
|
10916
|
+
currentHash: migrationHash(current)
|
|
10917
|
+
};
|
|
10918
|
+
}
|
|
10797
10919
|
function buildImportMigrationPlan(store, bundle, opts = {}) {
|
|
10798
10920
|
assertMigrationBundleIntegrity(bundle);
|
|
10799
10921
|
const includeRuns = opts.includeRuns ?? true;
|
|
@@ -10985,39 +11107,184 @@ async function requestJson(fetchImpl, config, path, init = {}) {
|
|
|
10985
11107
|
}
|
|
10986
11108
|
});
|
|
10987
11109
|
const payload = await response.json().catch(() => ({}));
|
|
10988
|
-
if (!response.ok)
|
|
10989
|
-
throw new Error(typeof payload.error === "string" ? payload.error : `loops-api request failed: ${response.status}`);
|
|
11110
|
+
if (!response.ok) {
|
|
11111
|
+
throw Object.assign(new Error(typeof payload.error === "string" ? payload.error : `loops-api request failed: ${response.status}`), { status: response.status, payload });
|
|
11112
|
+
}
|
|
10990
11113
|
return payload;
|
|
10991
11114
|
}
|
|
11115
|
+
async function fetchPagedRows(fetchImpl, config, path, key, opts) {
|
|
11116
|
+
const rows = [];
|
|
11117
|
+
const limit = 1000;
|
|
11118
|
+
for (let offset = 0;; offset += limit) {
|
|
11119
|
+
try {
|
|
11120
|
+
const separator = path.includes("?") ? "&" : "?";
|
|
11121
|
+
const payload = await requestJson(fetchImpl, config, `${path}${separator}limit=${limit}&offset=${offset}`);
|
|
11122
|
+
const page = Array.isArray(payload[key]) ? payload[key] : [];
|
|
11123
|
+
rows.push(...page);
|
|
11124
|
+
if (page.length < limit)
|
|
11125
|
+
break;
|
|
11126
|
+
} catch (error) {
|
|
11127
|
+
if (error.status === 404) {
|
|
11128
|
+
opts.unsupported.push(path);
|
|
11129
|
+
opts.warnings.push(`self-hosted control plane does not expose ${path}; exact ${key} comparison is unavailable`);
|
|
11130
|
+
break;
|
|
11131
|
+
}
|
|
11132
|
+
throw error;
|
|
11133
|
+
}
|
|
11134
|
+
}
|
|
11135
|
+
return rows;
|
|
11136
|
+
}
|
|
11137
|
+
async function fetchOptionalCount(fetchImpl, config, path, opts) {
|
|
11138
|
+
try {
|
|
11139
|
+
const payload = await requestJson(fetchImpl, config, path);
|
|
11140
|
+
return typeof payload.count === "number" ? payload.count : undefined;
|
|
11141
|
+
} catch (error) {
|
|
11142
|
+
if (error.status === 404) {
|
|
11143
|
+
opts.unsupported.push(path);
|
|
11144
|
+
opts.warnings.push(`self-hosted control plane does not expose ${path}; count comparison is unavailable`);
|
|
11145
|
+
return;
|
|
11146
|
+
}
|
|
11147
|
+
throw error;
|
|
11148
|
+
}
|
|
11149
|
+
}
|
|
10992
11150
|
async function fetchRemotePreview(opts) {
|
|
10993
11151
|
const config = resolveApiConfig(opts);
|
|
10994
11152
|
const warnings = [];
|
|
11153
|
+
const unsupported = [];
|
|
10995
11154
|
if (!config.apiUrl) {
|
|
10996
11155
|
warnings.push("LOOPS_API_URL or HASNA_LOOPS_API_URL is required to inspect a self-hosted control plane");
|
|
10997
|
-
return { loops: [], runs: [], warnings };
|
|
11156
|
+
return { workflows: [], loops: [], runs: [], counts: {}, unsupported, warnings };
|
|
10998
11157
|
}
|
|
10999
11158
|
if (!isLocalApiUrl(config.apiUrl) && !config.token) {
|
|
11000
11159
|
warnings.push("non-local self-hosted APIs require LOOPS_API_TOKEN or HASNA_LOOPS_API_TOKEN");
|
|
11001
|
-
return { loops: [], runs: [], warnings };
|
|
11160
|
+
return { workflows: [], loops: [], runs: [], counts: {}, unsupported, warnings };
|
|
11002
11161
|
}
|
|
11003
11162
|
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
11004
|
-
const
|
|
11005
|
-
const
|
|
11163
|
+
const api = { apiUrl: config.apiUrl, token: config.token };
|
|
11164
|
+
const requestOpts = { unsupported, warnings };
|
|
11165
|
+
const workflows = await fetchPagedRows(fetchImpl, api, "/v1/workflows", "workflows", requestOpts);
|
|
11166
|
+
const loops = await fetchPagedRows(fetchImpl, api, "/v1/loops?includeArchived=true", "loops", requestOpts);
|
|
11167
|
+
const runs = opts.includeRuns === false ? [] : await fetchPagedRows(fetchImpl, api, "/v1/runs?showOutput=true", "runs", requestOpts);
|
|
11006
11168
|
return {
|
|
11007
|
-
|
|
11008
|
-
|
|
11169
|
+
workflows,
|
|
11170
|
+
loops,
|
|
11171
|
+
runs,
|
|
11172
|
+
counts: {
|
|
11173
|
+
workflows: await fetchOptionalCount(fetchImpl, api, "/v1/workflows/count", requestOpts),
|
|
11174
|
+
loops: await fetchOptionalCount(fetchImpl, api, "/v1/loops/count?includeArchived=true", requestOpts),
|
|
11175
|
+
runs: opts.includeRuns === false ? undefined : await fetchOptionalCount(fetchImpl, api, "/v1/runs/count", requestOpts)
|
|
11176
|
+
},
|
|
11177
|
+
unsupported,
|
|
11009
11178
|
warnings
|
|
11010
11179
|
};
|
|
11011
11180
|
}
|
|
11181
|
+
function disabledWorkflowForSelfHostedImport(workflow) {
|
|
11182
|
+
return { ...workflow, status: "archived" };
|
|
11183
|
+
}
|
|
11184
|
+
function pausedLoopForSelfHostedImport(loop) {
|
|
11185
|
+
return {
|
|
11186
|
+
...loop,
|
|
11187
|
+
status: "paused",
|
|
11188
|
+
nextRunAt: undefined,
|
|
11189
|
+
retryScheduledFor: undefined
|
|
11190
|
+
};
|
|
11191
|
+
}
|
|
11192
|
+
function selfHostedDefinitionBundle(bundle) {
|
|
11193
|
+
const body = {
|
|
11194
|
+
...bundle,
|
|
11195
|
+
data: {
|
|
11196
|
+
workflows: bundle.data.workflows.map(disabledWorkflowForSelfHostedImport),
|
|
11197
|
+
loops: bundle.data.loops.map(pausedLoopForSelfHostedImport),
|
|
11198
|
+
runs: bundle.data.runs
|
|
11199
|
+
}
|
|
11200
|
+
};
|
|
11201
|
+
const { hash: _hash, ...hashBody } = body;
|
|
11202
|
+
return { ...body, hash: migrationHash(hashBody) };
|
|
11203
|
+
}
|
|
11204
|
+
function typedRows(rows) {
|
|
11205
|
+
return rows.filter((row) => Boolean(row && typeof row === "object" && typeof row.id === "string"));
|
|
11206
|
+
}
|
|
11207
|
+
function rowsById(rows) {
|
|
11208
|
+
return new Map(rows.map((row) => [row.id, row]));
|
|
11209
|
+
}
|
|
11210
|
+
function rowsByName(rows) {
|
|
11211
|
+
return new Map(rows.filter((row) => typeof row.name === "string").map((row) => [String(row.name), row]));
|
|
11212
|
+
}
|
|
11213
|
+
function rowIds(rows, resource, action) {
|
|
11214
|
+
return rows.filter((row) => row.resource === resource && row.action === action).map((row) => row.id);
|
|
11215
|
+
}
|
|
11216
|
+
function existingRowIds(rows, resource) {
|
|
11217
|
+
return rows.filter((row) => row.resource === resource && (row.action === "skip" || row.action === "update")).map((row) => row.id);
|
|
11218
|
+
}
|
|
11219
|
+
function buildSelfHostedManifest(args) {
|
|
11220
|
+
return {
|
|
11221
|
+
schema: LOOPS_SELF_HOSTED_PUSH_MANIFEST_SCHEMA,
|
|
11222
|
+
generatedAt: new Date().toISOString(),
|
|
11223
|
+
apiUrl: args.apiUrl,
|
|
11224
|
+
dryRun: args.dryRun,
|
|
11225
|
+
replace: args.replace,
|
|
11226
|
+
includeRuns: args.includeRuns,
|
|
11227
|
+
safety: {
|
|
11228
|
+
forcedLoopStatus: "paused",
|
|
11229
|
+
clearedLoopRunPointers: true,
|
|
11230
|
+
forcedWorkflowStatus: "archived",
|
|
11231
|
+
resumesLoops: false
|
|
11232
|
+
},
|
|
11233
|
+
counts: {
|
|
11234
|
+
source: args.bundle.counts,
|
|
11235
|
+
remoteBefore: args.remote.counts,
|
|
11236
|
+
remoteAfter: args.remoteAfter,
|
|
11237
|
+
plan: args.plan.summary,
|
|
11238
|
+
applied: args.applied,
|
|
11239
|
+
skipped: args.skipped,
|
|
11240
|
+
requests: args.requests
|
|
11241
|
+
},
|
|
11242
|
+
missingIds: {
|
|
11243
|
+
workflows: rowIds(args.plan.rows, "workflow", "insert"),
|
|
11244
|
+
loops: rowIds(args.plan.rows, "loop", "insert"),
|
|
11245
|
+
runs: rowIds(args.plan.rows, "run", "insert")
|
|
11246
|
+
},
|
|
11247
|
+
existingIds: {
|
|
11248
|
+
workflows: existingRowIds(args.plan.rows, "workflow"),
|
|
11249
|
+
loops: existingRowIds(args.plan.rows, "loop"),
|
|
11250
|
+
runs: existingRowIds(args.plan.rows, "run")
|
|
11251
|
+
},
|
|
11252
|
+
unsafe: {
|
|
11253
|
+
blocked: args.plan.rows.filter((row) => row.action === "blocked"),
|
|
11254
|
+
conflicts: args.plan.rows.filter((row) => row.action === "conflict"),
|
|
11255
|
+
unsupported: args.remote.unsupported,
|
|
11256
|
+
warnings: args.plan.warnings
|
|
11257
|
+
},
|
|
11258
|
+
rollback: {
|
|
11259
|
+
notes: [
|
|
11260
|
+
"self-hosted push imports through id-preserving upserts; no local loops are resumed or mutated",
|
|
11261
|
+
"imported loops are forced to paused with nextRunAt/retryScheduledFor cleared before upload",
|
|
11262
|
+
"imported workflows are forced to archived before upload",
|
|
11263
|
+
"rollback is manual: archive or delete imported ids from the self-hosted control plane after reviewing this manifest"
|
|
11264
|
+
],
|
|
11265
|
+
commands: [
|
|
11266
|
+
"loops self-hosted push --dry-run --no-runs --manifest-file <post-rollback-check.json>"
|
|
11267
|
+
]
|
|
11268
|
+
}
|
|
11269
|
+
};
|
|
11270
|
+
}
|
|
11012
11271
|
async function buildSelfHostedMigrationPlan(store, opts) {
|
|
11013
|
-
const bundle = exportLoopsMigrationBundle(store, { includeRuns: opts.includeRuns ?? true });
|
|
11272
|
+
const bundle = selfHostedDefinitionBundle(exportLoopsMigrationBundle(store, { includeRuns: opts.includeRuns ?? true }));
|
|
11014
11273
|
const remote = await fetchRemotePreview(opts);
|
|
11015
11274
|
const rows = [...bundle.blockers.map((row) => ({ ...row, action: "blocked" }))];
|
|
11016
11275
|
const warnings = [
|
|
11017
11276
|
...bundle.warnings,
|
|
11018
11277
|
...remote.warnings,
|
|
11019
|
-
"self-hosted
|
|
11278
|
+
"self-hosted push forces imported loops to paused and imported workflows to archived until activation is explicitly approved"
|
|
11020
11279
|
];
|
|
11280
|
+
const config = resolveApiConfig(opts);
|
|
11281
|
+
const apiUrl = config.apiUrl;
|
|
11282
|
+
if (!apiUrl) {
|
|
11283
|
+
pushBlocker(rows, "remote", "self-hosted-api-url", "LOOPS_API_URL or HASNA_LOOPS_API_URL is required to compare a self-hosted control plane");
|
|
11284
|
+
} else if (!isLocalApiUrl(apiUrl) && !config.token) {
|
|
11285
|
+
pushBlocker(rows, "remote", "self-hosted-api-token", "non-local self-hosted APIs require LOOPS_API_TOKEN or HASNA_LOOPS_API_TOKEN");
|
|
11286
|
+
}
|
|
11287
|
+
const replace = opts.replace ?? false;
|
|
11021
11288
|
if (opts.operation === "self-hosted-pull") {
|
|
11022
11289
|
for (const entry of remote.loops) {
|
|
11023
11290
|
const value = entry && typeof entry === "object" ? entry : {};
|
|
@@ -11047,58 +11314,107 @@ async function buildSelfHostedMigrationPlan(store, opts) {
|
|
|
11047
11314
|
schema: LOOPS_MIGRATION_SCHEMA,
|
|
11048
11315
|
operation: opts.operation,
|
|
11049
11316
|
dryRun: true,
|
|
11050
|
-
replace
|
|
11317
|
+
replace,
|
|
11051
11318
|
importable: false,
|
|
11052
11319
|
rows,
|
|
11053
11320
|
warnings
|
|
11054
11321
|
});
|
|
11055
|
-
return {
|
|
11322
|
+
return {
|
|
11323
|
+
...plan2,
|
|
11324
|
+
importable: false,
|
|
11325
|
+
manifest: buildSelfHostedManifest({
|
|
11326
|
+
apiUrl,
|
|
11327
|
+
dryRun: true,
|
|
11328
|
+
replace,
|
|
11329
|
+
includeRuns: opts.includeRuns ?? true,
|
|
11330
|
+
bundle,
|
|
11331
|
+
remote,
|
|
11332
|
+
plan: plan2
|
|
11333
|
+
})
|
|
11334
|
+
};
|
|
11056
11335
|
}
|
|
11057
|
-
const
|
|
11336
|
+
const remoteWorkflows = typedRows(remote.workflows);
|
|
11337
|
+
const remoteLoops = typedRows(remote.loops);
|
|
11338
|
+
const remoteRuns = typedRows(remote.runs);
|
|
11339
|
+
const remoteWorkflowsById = rowsById(remoteWorkflows);
|
|
11340
|
+
const remoteLoopsById = rowsById(remoteLoops);
|
|
11341
|
+
const remoteRunsById = rowsById(remoteRuns);
|
|
11342
|
+
const remoteLoopsByName = rowsByName(remoteLoops);
|
|
11058
11343
|
for (const workflow of bundle.data.workflows) {
|
|
11059
|
-
|
|
11344
|
+
if (remote.unsupported.some((entry) => entry.startsWith("/v1/workflows"))) {
|
|
11345
|
+
rows.push({
|
|
11346
|
+
resource: "workflow",
|
|
11347
|
+
id: workflow.id,
|
|
11348
|
+
name: workflow.name,
|
|
11349
|
+
action: "blocked",
|
|
11350
|
+
reason: "self-hosted API does not expose workflow list/count endpoints for safe comparison",
|
|
11351
|
+
incomingHash: migrationHash(workflow)
|
|
11352
|
+
});
|
|
11353
|
+
continue;
|
|
11354
|
+
}
|
|
11355
|
+
rows.push(remoteRepresentationRow(remoteWorkflowsById.get(workflow.id), workflow, {
|
|
11060
11356
|
resource: "workflow",
|
|
11061
11357
|
id: workflow.id,
|
|
11062
|
-
name: workflow.name
|
|
11063
|
-
|
|
11064
|
-
reason: "self-hosted API does not expose id-preserving workflow import/list endpoints yet",
|
|
11065
|
-
incomingHash: migrationHash(workflow)
|
|
11066
|
-
});
|
|
11358
|
+
name: workflow.name
|
|
11359
|
+
}));
|
|
11067
11360
|
}
|
|
11068
11361
|
for (const loop of bundle.data.loops) {
|
|
11069
|
-
const remoteLoop =
|
|
11070
|
-
|
|
11071
|
-
|
|
11072
|
-
id
|
|
11073
|
-
|
|
11074
|
-
|
|
11075
|
-
|
|
11076
|
-
|
|
11077
|
-
|
|
11078
|
-
|
|
11362
|
+
const remoteLoop = remoteLoopsById.get(loop.id);
|
|
11363
|
+
if (!remoteLoop) {
|
|
11364
|
+
const nameCollision = remoteLoopsByName.get(loop.name);
|
|
11365
|
+
if (nameCollision && nameCollision.id !== loop.id) {
|
|
11366
|
+
rows.push({
|
|
11367
|
+
resource: "loop",
|
|
11368
|
+
id: loop.id,
|
|
11369
|
+
name: loop.name,
|
|
11370
|
+
action: "insert",
|
|
11371
|
+
reason: `remote loop name exists under different id ${String(nameCollision.id ?? "unknown id")}; id-preserving import will add the missing source id paused`,
|
|
11372
|
+
incomingHash: migrationHash(loop),
|
|
11373
|
+
currentHash: migrationHash(nameCollision)
|
|
11374
|
+
});
|
|
11375
|
+
continue;
|
|
11376
|
+
}
|
|
11377
|
+
}
|
|
11378
|
+
rows.push(remoteRepresentationRow(remoteLoop, loop, { resource: "loop", id: loop.id, name: loop.name }));
|
|
11079
11379
|
}
|
|
11080
11380
|
if (opts.includeRuns ?? true) {
|
|
11081
11381
|
for (const run of bundle.data.runs) {
|
|
11082
|
-
|
|
11083
|
-
|
|
11084
|
-
|
|
11085
|
-
|
|
11086
|
-
|
|
11087
|
-
|
|
11088
|
-
|
|
11089
|
-
|
|
11382
|
+
if (run.status === "running") {
|
|
11383
|
+
rows.push({
|
|
11384
|
+
resource: "run",
|
|
11385
|
+
id: run.id,
|
|
11386
|
+
name: run.loopName,
|
|
11387
|
+
action: "blocked",
|
|
11388
|
+
reason: "running rows carry volatile lease/process ownership and are skipped by self-hosted import",
|
|
11389
|
+
incomingHash: migrationHash(run)
|
|
11390
|
+
});
|
|
11391
|
+
continue;
|
|
11392
|
+
}
|
|
11393
|
+
rows.push(remoteRepresentationRow(remoteRunsById.get(run.id), run, { resource: "run", id: run.id, name: run.loopName }));
|
|
11090
11394
|
}
|
|
11091
11395
|
}
|
|
11092
|
-
|
|
11396
|
+
let plan = finalizePlan({
|
|
11093
11397
|
schema: LOOPS_MIGRATION_SCHEMA,
|
|
11094
11398
|
operation: opts.operation,
|
|
11095
11399
|
dryRun: true,
|
|
11096
|
-
replace
|
|
11097
|
-
importable:
|
|
11400
|
+
replace,
|
|
11401
|
+
importable: bundle.importable && rows.every((row) => row.action !== "blocked" && row.action !== "conflict"),
|
|
11098
11402
|
rows,
|
|
11099
11403
|
warnings
|
|
11100
11404
|
});
|
|
11101
|
-
|
|
11405
|
+
plan = { ...plan, importable: plan.summary.blocked === 0 && plan.summary.conflict === 0 };
|
|
11406
|
+
return {
|
|
11407
|
+
...plan,
|
|
11408
|
+
manifest: buildSelfHostedManifest({
|
|
11409
|
+
apiUrl,
|
|
11410
|
+
dryRun: true,
|
|
11411
|
+
replace,
|
|
11412
|
+
includeRuns: opts.includeRuns ?? true,
|
|
11413
|
+
bundle,
|
|
11414
|
+
remote,
|
|
11415
|
+
plan
|
|
11416
|
+
})
|
|
11417
|
+
};
|
|
11102
11418
|
}
|
|
11103
11419
|
async function postImportBatch(fetchImpl, config, payload) {
|
|
11104
11420
|
const body = await requestJson(fetchImpl, config, "/v1/import", { method: "POST", body: JSON.stringify(payload) });
|
|
@@ -11119,23 +11435,29 @@ async function applySelfHostedPush(store, opts) {
|
|
|
11119
11435
|
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
11120
11436
|
const includeRuns = opts.includeRuns ?? true;
|
|
11121
11437
|
const replace = opts.replace ?? false;
|
|
11438
|
+
const plan = await buildSelfHostedMigrationPlan(store, { ...opts, operation: "self-hosted-push", includeRuns, replace });
|
|
11439
|
+
if (plan.summary.blocked > 0 || plan.summary.conflict > 0 || !plan.importable) {
|
|
11440
|
+
throw new ValidationError(`self-hosted push is not safe to apply: blocked=${plan.summary.blocked} conflict=${plan.summary.conflict}`);
|
|
11441
|
+
}
|
|
11122
11442
|
const batchRows = Math.max(1, opts.batchRows ?? 200);
|
|
11123
11443
|
const runBatchBytes = Math.max(64 * 1024, opts.runBatchBytes ?? 4 * 1024 * 1024);
|
|
11124
11444
|
const applied = { workflows: 0, loops: 0, runs: 0 };
|
|
11125
11445
|
const skipped = { runningRuns: 0, orphanRuns: 0 };
|
|
11126
11446
|
let requests = 0;
|
|
11127
11447
|
const base = store.exportMigrationRows({ includeRuns: false });
|
|
11448
|
+
const workflows = base.workflows.map(disabledWorkflowForSelfHostedImport);
|
|
11449
|
+
const loops = base.loops.map(pausedLoopForSelfHostedImport);
|
|
11128
11450
|
const loopIds = new Set(base.loops.map((loop) => loop.id));
|
|
11129
|
-
for (let i = 0;i <
|
|
11130
|
-
const batch =
|
|
11131
|
-
const result = await postImportBatch(fetchImpl, config, { workflows: batch, replace });
|
|
11451
|
+
for (let i = 0;i < workflows.length; i += batchRows) {
|
|
11452
|
+
const batch = workflows.slice(i, i + batchRows);
|
|
11453
|
+
const result = await postImportBatch(fetchImpl, config, { workflows: batch, replace, preserveWorkflowActivation: false, preserveLoopScheduling: false });
|
|
11132
11454
|
applied.workflows += result.imported.workflows;
|
|
11133
11455
|
requests += 1;
|
|
11134
11456
|
opts.onProgress?.({ phase: "workflows", sent: applied.workflows, requests });
|
|
11135
11457
|
}
|
|
11136
|
-
for (let i = 0;i <
|
|
11137
|
-
const batch =
|
|
11138
|
-
const result = await postImportBatch(fetchImpl, config, { loops: batch, replace });
|
|
11458
|
+
for (let i = 0;i < loops.length; i += batchRows) {
|
|
11459
|
+
const batch = loops.slice(i, i + batchRows);
|
|
11460
|
+
const result = await postImportBatch(fetchImpl, config, { loops: batch, replace, preserveLoopScheduling: false });
|
|
11139
11461
|
applied.loops += result.imported.loops;
|
|
11140
11462
|
requests += 1;
|
|
11141
11463
|
opts.onProgress?.({ phase: "loops", sent: applied.loops, requests });
|
|
@@ -11147,7 +11469,7 @@ async function applySelfHostedPush(store, opts) {
|
|
|
11147
11469
|
const flush = async () => {
|
|
11148
11470
|
if (pending.length === 0)
|
|
11149
11471
|
return;
|
|
11150
|
-
const result = await postImportBatch(fetchImpl, config, { runs: pending, replace });
|
|
11472
|
+
const result = await postImportBatch(fetchImpl, config, { runs: pending, replace, preserveLoopScheduling: false });
|
|
11151
11473
|
applied.runs += result.imported.runs;
|
|
11152
11474
|
skipped.runningRuns += result.skippedRunning;
|
|
11153
11475
|
requests += 1;
|
|
@@ -11177,7 +11499,36 @@ async function applySelfHostedPush(store, opts) {
|
|
|
11177
11499
|
}
|
|
11178
11500
|
await flush();
|
|
11179
11501
|
}
|
|
11180
|
-
|
|
11502
|
+
const remoteAfter = await fetchRemotePreview({ ...opts, operation: "self-hosted-push", includeRuns: false });
|
|
11503
|
+
const bundle = selfHostedDefinitionBundle(exportLoopsMigrationBundle(store, { includeRuns }));
|
|
11504
|
+
const remoteBefore = plan.manifest ? {
|
|
11505
|
+
workflows: [],
|
|
11506
|
+
loops: [],
|
|
11507
|
+
runs: [],
|
|
11508
|
+
counts: plan.manifest.counts.remoteBefore,
|
|
11509
|
+
unsupported: plan.manifest.unsafe.unsupported,
|
|
11510
|
+
warnings: []
|
|
11511
|
+
} : await fetchRemotePreview({ ...opts, operation: "self-hosted-push", includeRuns });
|
|
11512
|
+
return {
|
|
11513
|
+
ok: true,
|
|
11514
|
+
apiUrl: config.apiUrl,
|
|
11515
|
+
applied,
|
|
11516
|
+
skipped,
|
|
11517
|
+
requests,
|
|
11518
|
+
manifest: buildSelfHostedManifest({
|
|
11519
|
+
apiUrl: config.apiUrl,
|
|
11520
|
+
dryRun: false,
|
|
11521
|
+
replace,
|
|
11522
|
+
includeRuns,
|
|
11523
|
+
bundle,
|
|
11524
|
+
remote: remoteBefore,
|
|
11525
|
+
plan,
|
|
11526
|
+
applied,
|
|
11527
|
+
skipped,
|
|
11528
|
+
requests,
|
|
11529
|
+
remoteAfter: remoteAfter.counts
|
|
11530
|
+
})
|
|
11531
|
+
};
|
|
11181
11532
|
}
|
|
11182
11533
|
async function registerSelfHostedRunner(opts) {
|
|
11183
11534
|
const config = resolveApiConfig(opts);
|
|
@@ -14124,6 +14475,9 @@ var ROUTE_MANUAL_GATE_FIELDS = [
|
|
|
14124
14475
|
"approval_required",
|
|
14125
14476
|
"approvalRequired"
|
|
14126
14477
|
];
|
|
14478
|
+
function taskRouteDisallowedTag(tags) {
|
|
14479
|
+
return tags.find((tag) => ROUTE_DISALLOWED_TASK_TAGS.has(tag.toLowerCase()));
|
|
14480
|
+
}
|
|
14127
14481
|
function taskRouteEligibility(data, metadata) {
|
|
14128
14482
|
const records = taskEventRecords(data, metadata);
|
|
14129
14483
|
const automation = automationRecords(data, metadata);
|
|
@@ -15918,18 +16272,28 @@ function todosTaskRouteRedispatchBackoffMs(attempts) {
|
|
|
15918
16272
|
}
|
|
15919
16273
|
function reactivateStaleTodosTaskWorkItem(store, routeKey, item, now = Date.now()) {
|
|
15920
16274
|
if (routeKey !== "todos-task")
|
|
15921
|
-
return;
|
|
16275
|
+
return { kind: "dedupe" };
|
|
15922
16276
|
if (!REACTIVATABLE_TERMINAL_STATUSES.has(item.status))
|
|
15923
|
-
return;
|
|
15924
|
-
if (item.
|
|
15925
|
-
return;
|
|
16277
|
+
return { kind: "dedupe" };
|
|
16278
|
+
if (item.status === "dead_letter" && item.gateDeaths >= GATE_DEATH_CEILING) {
|
|
16279
|
+
return { kind: "dedupe" };
|
|
16280
|
+
}
|
|
16281
|
+
if (item.attempts >= MAX_TODOS_TASK_ROUTE_REDISPATCHES) {
|
|
16282
|
+
if (item.status === "dead_letter")
|
|
16283
|
+
return { kind: "dedupe" };
|
|
16284
|
+
const deadLettered = store.deadLetterWorkflowWorkItem(item.id, {
|
|
16285
|
+
reason: `redispatch cap reached (${item.attempts}/${MAX_TODOS_TASK_ROUTE_REDISPATCHES}): todos task still actionable but ` + `${item.attempts} runs finished without closing it; dead-lettered. Fix the task or 'loops routes requeue' to retry (resets attempts).`
|
|
16286
|
+
});
|
|
16287
|
+
return { kind: "dead-letter", item: deadLettered };
|
|
16288
|
+
}
|
|
15926
16289
|
const finishedAt = Date.parse(item.updatedAt);
|
|
15927
16290
|
if (Number.isFinite(finishedAt) && now - finishedAt < todosTaskRouteRedispatchBackoffMs(item.attempts)) {
|
|
15928
|
-
return;
|
|
16291
|
+
return { kind: "dedupe" };
|
|
15929
16292
|
}
|
|
15930
|
-
|
|
16293
|
+
const requeued = store.requeueWorkflowWorkItem(item.id, {
|
|
15931
16294
|
reason: `re-admitted from ${item.status}: todos task still actionable after prior run (attempt ${item.attempts + 1}/${MAX_TODOS_TASK_ROUTE_REDISPATCHES})`
|
|
15932
16295
|
});
|
|
16296
|
+
return { kind: "readmit", item: requeued };
|
|
15933
16297
|
}
|
|
15934
16298
|
function findRouteWorkItemByKeys(store, routeKey, idempotencyKeys) {
|
|
15935
16299
|
const [primaryKey, ...aliasKeys] = idempotencyKeys;
|
|
@@ -16034,12 +16398,14 @@ function codewithAuthProfilesFromWorkflow(workflow) {
|
|
|
16034
16398
|
return profiles.length ? profiles : undefined;
|
|
16035
16399
|
}
|
|
16036
16400
|
function dedupedRoutePrint(plan, outcome) {
|
|
16401
|
+
const deadLettered = outcome.existingItem.status === "dead_letter";
|
|
16037
16402
|
return {
|
|
16038
16403
|
kind: "deduped",
|
|
16039
16404
|
value: {
|
|
16040
16405
|
deduped: true,
|
|
16041
16406
|
idempotencyKey: plan.idempotencyKey,
|
|
16042
16407
|
dedupedBy: "work-item",
|
|
16408
|
+
...deadLettered ? { deadLettered: true, reason: outcome.existingItem.lastReason } : {},
|
|
16043
16409
|
event: plan.event,
|
|
16044
16410
|
...plan.dedupeValueExtras,
|
|
16045
16411
|
invocation: outcome.invocation ? publicWorkflowInvocation(outcome.invocation) : undefined,
|
|
@@ -16047,7 +16413,7 @@ function dedupedRoutePrint(plan, outcome) {
|
|
|
16047
16413
|
workflow: outcome.existingWorkflow ? publicWorkflow(outcome.existingWorkflow) : undefined,
|
|
16048
16414
|
loop: outcome.existingLoop ? publicLoop(outcome.existingLoop) : undefined
|
|
16049
16415
|
},
|
|
16050
|
-
human: `deduped existing work item ${outcome.existingItem.id} for event=${plan.event.id} idempotency=${plan.idempotencyKey}`
|
|
16416
|
+
human: deadLettered ? `dead-lettered work item ${outcome.existingItem.id} (redispatch cap reached) for event=${plan.event.id} idempotency=${plan.idempotencyKey}` : `deduped existing work item ${outcome.existingItem.id} for event=${plan.event.id} idempotency=${plan.idempotencyKey}`
|
|
16051
16417
|
};
|
|
16052
16418
|
}
|
|
16053
16419
|
function routeEvent(plan) {
|
|
@@ -16136,10 +16502,12 @@ function routeEvent(plan) {
|
|
|
16136
16502
|
const invocation = store.createWorkflowInvocation(plan.invocationInput);
|
|
16137
16503
|
const existingItem = findRouteWorkItemByKeys(store, plan.routeKey, dedupeKeys);
|
|
16138
16504
|
if (existingItem) {
|
|
16139
|
-
|
|
16140
|
-
|
|
16141
|
-
const
|
|
16142
|
-
|
|
16505
|
+
const reactivation = reactivateStaleTodosTaskWorkItem(store, plan.routeKey, existingItem);
|
|
16506
|
+
if (reactivation.kind !== "readmit") {
|
|
16507
|
+
const dedupeItem = reactivation.kind === "dead-letter" ? reactivation.item : existingItem;
|
|
16508
|
+
const existingLoop = dedupeItem.loopId ? store.getLoop(dedupeItem.loopId) : undefined;
|
|
16509
|
+
const existingWorkflow = dedupeItem.workflowId ? store.getWorkflow(dedupeItem.workflowId) : undefined;
|
|
16510
|
+
return { kind: "deduped", existingItem: dedupeItem, existingLoop, existingWorkflow, invocation };
|
|
16143
16511
|
}
|
|
16144
16512
|
}
|
|
16145
16513
|
const throttle = hasThrottleLimits(plan.throttleLimits) ? routeThrottleDecision(store, {
|
|
@@ -16305,11 +16673,13 @@ function routeTodosTaskEvent(event, opts) {
|
|
|
16305
16673
|
try {
|
|
16306
16674
|
const existingItem = findRouteWorkItemByKeys(store, "todos-task", [idempotencyKey, ...dedupeAliases]);
|
|
16307
16675
|
if (existingItem) {
|
|
16308
|
-
|
|
16309
|
-
|
|
16310
|
-
const
|
|
16311
|
-
const
|
|
16312
|
-
|
|
16676
|
+
const reactivation = reactivateStaleTodosTaskWorkItem(store, "todos-task", existingItem);
|
|
16677
|
+
if (reactivation.kind !== "readmit") {
|
|
16678
|
+
const dedupeItem = reactivation.kind === "dead-letter" ? reactivation.item : existingItem;
|
|
16679
|
+
const existingLoop = dedupeItem.loopId ? store.getLoop(dedupeItem.loopId) : undefined;
|
|
16680
|
+
const existingWorkflow = dedupeItem.workflowId ? store.getWorkflow(dedupeItem.workflowId) : undefined;
|
|
16681
|
+
const existingInvocation = store.getWorkflowInvocation(dedupeItem.invocationId);
|
|
16682
|
+
return dedupedRoutePrint({ event, idempotencyKey, dedupeValueExtras: {} }, { existingItem: dedupeItem, existingLoop, existingWorkflow, invocation: existingInvocation });
|
|
16313
16683
|
}
|
|
16314
16684
|
}
|
|
16315
16685
|
} finally {
|
|
@@ -16622,6 +16992,33 @@ function taskSourceWorkingDir(task) {
|
|
|
16622
16992
|
function canonicalRoutePath(path) {
|
|
16623
16993
|
return normalizeRoutePath(path) ?? (path?.trim() ? resolve8(path) : undefined);
|
|
16624
16994
|
}
|
|
16995
|
+
function taskUsableRepoProjectPath(task, isRepo) {
|
|
16996
|
+
const metadata = objectField2(task.metadata) ?? {};
|
|
16997
|
+
const candidates = [
|
|
16998
|
+
taskField(task, ["project_path", "projectPath"]),
|
|
16999
|
+
taskEventField(metadata, ["project_path", "projectPath", "project_canonical_path"]),
|
|
17000
|
+
taskDescriptionProjectPath(task),
|
|
17001
|
+
taskField(task, ["working_dir", "workingDir", "cwd"]),
|
|
17002
|
+
taskEventField(metadata, ["working_dir", "workingDir", "cwd"])
|
|
17003
|
+
];
|
|
17004
|
+
for (const candidate of candidates) {
|
|
17005
|
+
const canonical = canonicalRoutePath(candidate);
|
|
17006
|
+
if (canonical && isRepo(canonical))
|
|
17007
|
+
return canonical;
|
|
17008
|
+
}
|
|
17009
|
+
return;
|
|
17010
|
+
}
|
|
17011
|
+
function memoizedIsExistingGitProjectPath() {
|
|
17012
|
+
const cache = new Map;
|
|
17013
|
+
return (path) => {
|
|
17014
|
+
const cached = cache.get(path);
|
|
17015
|
+
if (cached !== undefined)
|
|
17016
|
+
return cached;
|
|
17017
|
+
const result = isExistingGitProjectPath(path);
|
|
17018
|
+
cache.set(path, result);
|
|
17019
|
+
return result;
|
|
17020
|
+
};
|
|
17021
|
+
}
|
|
16625
17022
|
function taskListValues(task) {
|
|
16626
17023
|
const taskList = objectField2(task.task_list);
|
|
16627
17024
|
return [
|
|
@@ -16737,6 +17134,7 @@ function compactDrainResult(result) {
|
|
|
16737
17134
|
routeScope: stringField(value.routeScope),
|
|
16738
17135
|
requeue,
|
|
16739
17136
|
queuedAtSource: value.queuedAtSource,
|
|
17137
|
+
deadLettered: value.deadLettered === true ? true : undefined,
|
|
16740
17138
|
fatal: value.fatal === true ? true : undefined
|
|
16741
17139
|
};
|
|
16742
17140
|
}
|
|
@@ -17015,10 +17413,12 @@ function drainTodosTaskRoutes(opts) {
|
|
|
17015
17413
|
scanned: 0,
|
|
17016
17414
|
candidates: 0,
|
|
17017
17415
|
filteredCandidates: 0,
|
|
17416
|
+
excludedDisallowedTag: 0,
|
|
17018
17417
|
scanExhausted: false,
|
|
17019
17418
|
considered: 0,
|
|
17020
17419
|
created: 0,
|
|
17021
17420
|
deduped: 0,
|
|
17421
|
+
deadLettered: 0,
|
|
17022
17422
|
throttled: 0,
|
|
17023
17423
|
skipped: 0,
|
|
17024
17424
|
freshnessClosed: 0,
|
|
@@ -17047,10 +17447,12 @@ function drainTodosTaskRoutes(opts) {
|
|
|
17047
17447
|
scanned: report2.scanned,
|
|
17048
17448
|
candidates: report2.candidates,
|
|
17049
17449
|
filteredCandidates: report2.filteredCandidates,
|
|
17450
|
+
excludedDisallowedTag: report2.excludedDisallowedTag,
|
|
17050
17451
|
scanExhausted: report2.scanExhausted,
|
|
17051
17452
|
considered: report2.considered,
|
|
17052
17453
|
created: report2.created,
|
|
17053
17454
|
deduped: report2.deduped,
|
|
17455
|
+
deadLettered: report2.deadLettered,
|
|
17054
17456
|
throttled: report2.throttled,
|
|
17055
17457
|
skipped: report2.skipped,
|
|
17056
17458
|
freshnessClosed: report2.freshnessClosed,
|
|
@@ -17076,7 +17478,14 @@ function drainTodosTaskRoutes(opts) {
|
|
|
17076
17478
|
projectPathPrefix: opts.projectPathPrefix,
|
|
17077
17479
|
tags: requiredTags
|
|
17078
17480
|
}));
|
|
17079
|
-
const
|
|
17481
|
+
const requiredTagSet = new Set(requiredTags.map((tag) => tag.toLowerCase()));
|
|
17482
|
+
const windowCandidates = filteredCandidates.filter((task) => {
|
|
17483
|
+
const disallowed = taskRouteDisallowedTag(tagsFromValue2(task.tags));
|
|
17484
|
+
return !disallowed || requiredTagSet.has(disallowed.toLowerCase());
|
|
17485
|
+
});
|
|
17486
|
+
const excludedDisallowedTag = filteredCandidates.length - windowCandidates.length;
|
|
17487
|
+
const candidates = windowCandidates.slice(0, candidateLimit);
|
|
17488
|
+
const isRepo = memoizedIsExistingGitProjectPath();
|
|
17080
17489
|
const results = [];
|
|
17081
17490
|
let created = 0;
|
|
17082
17491
|
for (const task of candidates) {
|
|
@@ -17086,7 +17495,7 @@ function drainTodosTaskRoutes(opts) {
|
|
|
17086
17495
|
let result;
|
|
17087
17496
|
try {
|
|
17088
17497
|
const sourceProject = opts.todosProjectsFromRegistry ? taskField(task, ["source_project_path", "sourceProjectPath"]) : undefined;
|
|
17089
|
-
const explicitRouteProjectPath = sourceProject ? taskRoutePathFromRegistry(sourceProject) : opts.projectPath;
|
|
17498
|
+
const explicitRouteProjectPath = sourceProject ? taskRoutePathFromRegistry(sourceProject) : taskUsableRepoProjectPath(task, isRepo) ?? opts.projectPath;
|
|
17090
17499
|
event = taskDrainEvent(task, sourceProject, explicitRouteProjectPath);
|
|
17091
17500
|
result = routeTodosTaskEvent(event, {
|
|
17092
17501
|
...opts,
|
|
@@ -17129,10 +17538,12 @@ function drainTodosTaskRoutes(opts) {
|
|
|
17129
17538
|
scanned: ready.length,
|
|
17130
17539
|
candidates: candidates.length,
|
|
17131
17540
|
filteredCandidates: filteredCandidates.length,
|
|
17132
|
-
|
|
17541
|
+
excludedDisallowedTag,
|
|
17542
|
+
scanExhausted: ready.length >= scanLimit && windowCandidates.length < candidateLimit,
|
|
17133
17543
|
considered: results.length,
|
|
17134
17544
|
created: results.filter((result) => result.kind === "created" && !result.value.deduped).length,
|
|
17135
17545
|
deduped: results.filter((result) => result.kind === "deduped").length,
|
|
17546
|
+
deadLettered: results.filter((result) => result.kind === "deduped" && result.value.deadLettered === true).length,
|
|
17136
17547
|
throttled: results.filter((result) => result.kind === "throttled").length,
|
|
17137
17548
|
skipped: results.filter((result) => result.kind === "skipped").length,
|
|
17138
17549
|
freshnessClosed: results.filter((result) => isFreshnessSkip(result) && result.value.sourceTaskUpdate?.attempted === true).length,
|
|
@@ -17161,10 +17572,12 @@ function drainTodosTaskRoutes(opts) {
|
|
|
17161
17572
|
scanned: report.scanned,
|
|
17162
17573
|
candidates: report.candidates,
|
|
17163
17574
|
filteredCandidates: report.filteredCandidates,
|
|
17575
|
+
excludedDisallowedTag: report.excludedDisallowedTag,
|
|
17164
17576
|
scanExhausted: report.scanExhausted,
|
|
17165
17577
|
considered: report.considered,
|
|
17166
17578
|
created: report.created,
|
|
17167
17579
|
deduped: report.deduped,
|
|
17580
|
+
deadLettered: report.deadLettered,
|
|
17168
17581
|
throttled: report.throttled,
|
|
17169
17582
|
skipped: report.skipped,
|
|
17170
17583
|
freshnessClosed: report.freshnessClosed,
|
|
@@ -17179,7 +17592,7 @@ function drainTodosTaskRoutes(opts) {
|
|
|
17179
17592
|
} : { ...report, evidencePath };
|
|
17180
17593
|
return {
|
|
17181
17594
|
value,
|
|
17182
|
-
human: `drained todos ready queue: considered=${report.considered} created=${report.created} deduped=${report.deduped} throttled=${report.throttled} skipped=${report.skipped} freshnessClosed=${report.freshnessClosed} fatal=${report.fatal}`
|
|
17595
|
+
human: `drained todos ready queue: considered=${report.considered} created=${report.created} deduped=${report.deduped} deadLettered=${report.deadLettered} throttled=${report.throttled} skipped=${report.skipped} freshnessClosed=${report.freshnessClosed} fatal=${report.fatal} excludedDisallowedTag=${report.excludedDisallowedTag}`
|
|
17183
17596
|
};
|
|
17184
17597
|
}
|
|
17185
17598
|
// src/lib/route/route-tasks.ts
|
|
@@ -17846,6 +18259,14 @@ function printMigrationPlan(plan, opts = {}) {
|
|
|
17846
18259
|
console.log(`${row.action} ${row.resource}:${row.name ?? row.id} ${row.reason ?? ""}`.trim());
|
|
17847
18260
|
}
|
|
17848
18261
|
}
|
|
18262
|
+
function writeManifestFile(file, manifest) {
|
|
18263
|
+
if (!file)
|
|
18264
|
+
return;
|
|
18265
|
+
if (manifest === undefined)
|
|
18266
|
+
throw new ValidationError("command did not produce a manifest");
|
|
18267
|
+
writeFileSync7(file, `${JSON.stringify(manifest, null, 2)}
|
|
18268
|
+
`, { mode: 384 });
|
|
18269
|
+
}
|
|
17849
18270
|
function backupLoopsDatabase(reason) {
|
|
17850
18271
|
return backupDatabase({ reason, keep: 3 }).path;
|
|
17851
18272
|
}
|
|
@@ -18003,15 +18424,17 @@ function selfHostedMigrationCommand(operation) {
|
|
|
18003
18424
|
});
|
|
18004
18425
|
}
|
|
18005
18426
|
selfHosted.command("migrate").description("preview local-to-self-hosted migration actions").option("--api-url <url>", "self-hosted control-plane API URL").option("--dry-run", "preview only; self-hosted migrate does not apply remote changes yet").option("--no-runs", "omit loop run history from the preview").option("--json", "print JSON").action(selfHostedMigrationCommand("self-hosted-migrate"));
|
|
18006
|
-
selfHosted.command("push").description("preview (default) or apply an id-preserving local->self-hosted backfill").option("--api-url <url>", "self-hosted control-plane API URL").option("--apply", "apply the backfill via the control-plane /v1/import endpoint (default is preview)").option("--replace", "update
|
|
18427
|
+
selfHosted.command("push").description("preview (default) or apply an id-preserving local->self-hosted backfill").option("--api-url <url>", "self-hosted control-plane API URL").option("--apply", "apply the backfill via the control-plane /v1/import endpoint (default is preview)").option("--replace", "update differing same-id remote rows; safe default may still archive/pause same-id definitions").option("--dry-run", "preview only; equivalent to omitting --apply").option("--no-runs", "omit loop run history").option("--manifest-file <path>", "write a self-hosted comparison/import manifest JSON file").option("--json", "print JSON").action(runAction(async (opts) => {
|
|
18007
18428
|
if (!opts.apply || opts.dryRun) {
|
|
18008
18429
|
const store2 = new Store;
|
|
18009
18430
|
try {
|
|
18010
18431
|
const plan = await buildSelfHostedMigrationPlan(store2, {
|
|
18011
18432
|
operation: "self-hosted-push",
|
|
18012
18433
|
apiUrl: opts.apiUrl,
|
|
18013
|
-
includeRuns: opts.runs
|
|
18434
|
+
includeRuns: opts.runs,
|
|
18435
|
+
replace: opts.replace
|
|
18014
18436
|
});
|
|
18437
|
+
writeManifestFile(opts.manifestFile, plan.manifest);
|
|
18015
18438
|
printMigrationPlan(plan, opts);
|
|
18016
18439
|
} finally {
|
|
18017
18440
|
store2.close();
|
|
@@ -18025,6 +18448,7 @@ selfHosted.command("push").description("preview (default) or apply an id-preserv
|
|
|
18025
18448
|
includeRuns: opts.runs,
|
|
18026
18449
|
replace: opts.replace
|
|
18027
18450
|
});
|
|
18451
|
+
writeManifestFile(opts.manifestFile, result.manifest);
|
|
18028
18452
|
if (isJson() || opts.json)
|
|
18029
18453
|
console.log(JSON.stringify(result, null, 2));
|
|
18030
18454
|
else {
|
|
@@ -18194,12 +18618,12 @@ routes.command("show <id>").description("show one admission work item").action(r
|
|
|
18194
18618
|
loop: loop ? publicLoop(loop) : undefined
|
|
18195
18619
|
}, `${item.id} ${item.status} ${item.routeKey} ${item.subjectRef}`);
|
|
18196
18620
|
})));
|
|
18197
|
-
routes.command("requeue <id>").description("requeue a terminal admission work item for the next task/event delivery").option("--reason <text>", "operator reason recorded on the work item").action(runAction((id, opts) => withStore(async (store) => {
|
|
18621
|
+
routes.command("requeue <id>").description("requeue a terminal admission work item for the next task/event delivery (resets the redispatch attempt count so the unwedge is durable; --keep-attempts preserves it)").option("--reason <text>", "operator reason recorded on the work item").option("--keep-attempts", "preserve the redispatch attempt count instead of resetting it (cautious path: the item may re-cap after one more terminal run)").action(runAction((id, opts) => withStore(async (store) => {
|
|
18198
18622
|
const reason = stringField(opts.reason);
|
|
18199
18623
|
if (!reason)
|
|
18200
18624
|
throw new ValidationError("routes requeue requires --reason <text>");
|
|
18201
|
-
const item = await store.requeueWorkflowWorkItem(id, { reason });
|
|
18202
|
-
print(publicWorkflowWorkItem(item), `requeued route work item ${item.id} (${item.routeKey})`);
|
|
18625
|
+
const item = await store.requeueWorkflowWorkItem(id, { reason, resetAttempts: !opts.keepAttempts });
|
|
18626
|
+
print(publicWorkflowWorkItem(item), `requeued route work item ${item.id} (${item.routeKey}) - ${opts.keepAttempts ? "attempts preserved" : "attempts reset"}`);
|
|
18203
18627
|
})));
|
|
18204
18628
|
routes.command("invocations").description("list workflow invocations").option("--limit <n>", "maximum rows", "50").action(runAction((opts) => withStore(async (store) => {
|
|
18205
18629
|
const invocations = await store.listWorkflowInvocations({ limit: positiveInteger(opts.limit, "--limit") ?? 50 });
|