@hasna/loops 0.4.27 → 0.4.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +205 -1
- package/README.md +232 -49
- package/dist/api/index.d.ts +20 -19
- package/dist/api/index.js +7770 -1342
- package/dist/cli/index.js +2976 -966
- package/dist/cli/safe-error-context.d.ts +1 -0
- package/dist/daemon/daemon.d.ts +1 -0
- package/dist/daemon/index.js +1899 -499
- package/dist/generated/storage-kit/index.d.ts +1 -1
- package/dist/generated/storage-kit/mode.d.ts +6 -12
- package/dist/index.d.ts +3 -3
- package/dist/index.js +5319 -1096
- package/dist/lib/advancement.d.ts +52 -0
- package/dist/lib/agent-adapter.d.ts +20 -2
- package/dist/lib/auth/route-policy.d.ts +14 -0
- package/dist/lib/auth/tenant-auth.d.ts +38 -0
- package/dist/lib/cloud/mode.d.ts +2 -8
- package/dist/lib/cloud/storage.d.ts +1 -2
- package/dist/lib/cloud/transport.d.ts +4 -18
- package/dist/lib/errors.d.ts +43 -1
- package/dist/lib/executor.d.ts +9 -0
- package/dist/lib/format.d.ts +2 -2
- package/dist/lib/goal/runner.d.ts +36 -3
- package/dist/lib/health.d.ts +1 -1
- package/dist/lib/labels.d.ts +4 -0
- package/dist/lib/loop-status.d.ts +4 -0
- package/dist/lib/migration.d.ts +70 -17
- package/dist/lib/mode.d.ts +2 -5
- package/dist/lib/mode.js +28 -37
- package/dist/lib/route/fields.d.ts +8 -0
- package/dist/lib/route/index.d.ts +2 -2
- package/dist/lib/route/throttle.d.ts +7 -0
- package/dist/lib/route/types.d.ts +3 -0
- package/dist/lib/run-completion.d.ts +18 -0
- package/dist/lib/scheduler.d.ts +5 -11
- package/dist/lib/storage/contract.d.ts +15 -1
- package/dist/lib/storage/index.d.ts +1 -1
- package/dist/lib/storage/index.js +4083 -486
- package/dist/lib/storage/pg-executor.d.ts +10 -5
- package/dist/lib/storage/postgres-loop-storage.d.ts +52 -26
- package/dist/lib/storage/postgres-schema.d.ts +8 -0
- package/dist/lib/storage/postgres-schema.js +1326 -1
- package/dist/lib/storage/postgres.d.ts +3 -1
- package/dist/lib/storage/postgres.js +1356 -27
- package/dist/lib/storage/provider-credentials.d.ts +84 -0
- package/dist/lib/storage/shared-database-transfer.d.ts +136 -0
- package/dist/lib/storage/sqlite.d.ts +15 -2
- package/dist/lib/storage/sqlite.js +1379 -217
- package/dist/lib/storage/tenant-backfill-s3.d.ts +53 -0
- package/dist/lib/storage/tenant-backfill.d.ts +40 -0
- package/dist/lib/store/index.d.ts +14 -8
- package/dist/lib/store.d.ts +129 -7
- package/dist/lib/store.js +1352 -218
- package/dist/lib/templates.d.ts +11 -0
- package/dist/lib/workflow-events.d.ts +6 -0
- package/dist/lib/workflow-provenance.d.ts +8 -0
- package/dist/lib/workflow-runner.d.ts +52 -4
- package/dist/mcp/index.js +2074 -657
- package/dist/runner/index.d.ts +20 -2
- package/dist/runner/index.js +2146 -183
- package/dist/sdk/http.d.ts +364 -4
- package/dist/sdk/http.js +379 -1
- package/dist/sdk/index.d.ts +7 -2
- package/dist/sdk/index.js +2341 -742
- package/dist/serve/index.d.ts +14 -0
- package/dist/serve/index.js +13269 -1830
- package/dist/types.d.ts +75 -3
- package/docs/AUTOMATION_RUNTIME_DESIGN.md +32 -32
- package/docs/CUTOVER-RUNBOOK.md +158 -31
- package/docs/DEPLOYMENT_MODES.md +84 -56
- package/docs/RUNTIME_BOUNDARY.md +26 -26
- package/docs/SHARED-DATABASE-TRANSFER.md +95 -0
- package/docs/SHARED_KIT_EXTRACTION_INVENTORY.md +631 -0
- package/docs/TRANSCRIPT_LOOP_PATTERNS.md +3 -3
- package/docs/UNIFIED_PRODUCT_CONTRACT.md +365 -0
- package/docs/USAGE.md +150 -56
- package/docs/workflows/transcript-feedback-to-loops.json +2 -2
- package/package.json +8 -4
- package/dist/lib/storage/pg-runner-claim.d.ts +0 -40
package/dist/types.d.ts
CHANGED
|
@@ -2,6 +2,12 @@ import type { GoalSpec } from "./lib/goal/types.js";
|
|
|
2
2
|
export type { Goal, GoalAutoExecute, GoalExecutorResult, GoalPlan, GoalPlanNode, GoalPlanNodeStatus, GoalPlanStatus, GoalRollup, GoalRun, GoalSpec, GoalStatus, } from "./lib/goal/types.js";
|
|
3
3
|
export type LoopStatus = "active" | "paused" | "stopped" | "expired";
|
|
4
4
|
export type RunStatus = "running" | "succeeded" | "failed" | "timed_out" | "abandoned" | "skipped";
|
|
5
|
+
export interface RecoveredLeaseRunSnapshotEntry {
|
|
6
|
+
updatedAt: string;
|
|
7
|
+
scheduledFor: string;
|
|
8
|
+
id: string;
|
|
9
|
+
attempt: number;
|
|
10
|
+
}
|
|
5
11
|
export type CatchUpPolicy = "none" | "latest" | "all";
|
|
6
12
|
export type OverlapPolicy = "skip" | "allow";
|
|
7
13
|
export type IntervalAnchor = "fixed_rate" | "fixed_delay";
|
|
@@ -80,10 +86,13 @@ export type AgentProvider = "claude" | "cursor" | "codewith" | "aicopilot" | "op
|
|
|
80
86
|
export type AgentConfigIsolation = "safe" | "none";
|
|
81
87
|
export type AgentPermissionMode = "default" | "plan" | "auto" | "bypass";
|
|
82
88
|
export type AgentSandbox = "read-only" | "workspace-write" | "danger-full-access" | "enabled" | "disabled";
|
|
89
|
+
export type AgentAllowlistEnforcement = "metadata_only";
|
|
83
90
|
export interface AgentAllowlistSpec {
|
|
91
|
+
/** Advisory provider metadata. Restrictions require this non-empty audit reason. */
|
|
84
92
|
tools?: string[];
|
|
85
93
|
commands?: string[];
|
|
86
|
-
enforcement?:
|
|
94
|
+
enforcement?: AgentAllowlistEnforcement;
|
|
95
|
+
safetyReason?: string;
|
|
87
96
|
}
|
|
88
97
|
export type AgentWorktreeMode = "auto" | "required" | "off" | "main";
|
|
89
98
|
export interface AgentWorktreeSpec {
|
|
@@ -108,6 +117,25 @@ export interface AgentRoutingSpec {
|
|
|
108
117
|
* for per-account attribution and least-loaded auth-profile pool selection. */
|
|
109
118
|
role?: "triage" | "planner" | "worker" | "verifier";
|
|
110
119
|
}
|
|
120
|
+
/** Server-derived audit contract for an agent step inside a workflow run. */
|
|
121
|
+
export interface AgentSessionContract {
|
|
122
|
+
version: 1;
|
|
123
|
+
provider: AgentProvider;
|
|
124
|
+
model?: string;
|
|
125
|
+
cwd?: string;
|
|
126
|
+
permissionMode: AgentPermissionMode;
|
|
127
|
+
sandbox: AgentSandbox | "provider-default";
|
|
128
|
+
manualBreakGlass: boolean;
|
|
129
|
+
routing?: AgentRoutingSpec;
|
|
130
|
+
timeoutMs: TimeoutMs;
|
|
131
|
+
restrictions: {
|
|
132
|
+
tools?: string[];
|
|
133
|
+
commands?: string[];
|
|
134
|
+
enforcement: AgentAllowlistEnforcement;
|
|
135
|
+
providerEnforced: false;
|
|
136
|
+
};
|
|
137
|
+
safetyReason?: string;
|
|
138
|
+
}
|
|
111
139
|
export interface AgentPromptSource {
|
|
112
140
|
type: "file";
|
|
113
141
|
path: string;
|
|
@@ -120,6 +148,11 @@ export interface AgentTargetBase {
|
|
|
120
148
|
variant?: string;
|
|
121
149
|
agent?: string;
|
|
122
150
|
authProfile?: string;
|
|
151
|
+
/**
|
|
152
|
+
* Provider CLI passthrough arguments. Fail-closed: omitted or empty is valid,
|
|
153
|
+
* while every non-empty or malformed entry is rejected until that exact
|
|
154
|
+
* provider option is explicitly reviewed and allowlisted by the adapter.
|
|
155
|
+
*/
|
|
123
156
|
extraArgs?: string[];
|
|
124
157
|
addDirs?: string[];
|
|
125
158
|
timeoutMs?: TimeoutMs;
|
|
@@ -127,6 +160,8 @@ export interface AgentTargetBase {
|
|
|
127
160
|
configIsolation?: AgentConfigIsolation;
|
|
128
161
|
permissionMode?: AgentPermissionMode;
|
|
129
162
|
sandbox?: AgentSandbox;
|
|
163
|
+
/** Explicit operator acknowledgement required with a non-empty safetyReason for relaxed sandbox or provider bypass modes. */
|
|
164
|
+
manualBreakGlass?: boolean;
|
|
130
165
|
allowlist?: AgentAllowlistSpec;
|
|
131
166
|
worktree?: AgentWorktreeSpec;
|
|
132
167
|
routing?: AgentRoutingSpec;
|
|
@@ -223,6 +258,16 @@ export interface WorkflowWorkItem {
|
|
|
223
258
|
priority: number;
|
|
224
259
|
status: WorkflowWorkItemStatus;
|
|
225
260
|
attempts: number;
|
|
261
|
+
/**
|
|
262
|
+
* Consecutive non-productive "gate death" finishes (runs that failed at
|
|
263
|
+
* worktree prep or a fast triage/planner gate before any real work). Gate
|
|
264
|
+
* deaths refund their redispatch attempt, so this second counter bounds a
|
|
265
|
+
* deterministic infrastructure fault: at the ceiling the item is
|
|
266
|
+
* dead-lettered instead of retrying forever. Reset by a run that reaches
|
|
267
|
+
* the worker (success, productive failure, or tempfail) and by an
|
|
268
|
+
* operator requeue with attempts reset.
|
|
269
|
+
*/
|
|
270
|
+
gateDeaths: number;
|
|
226
271
|
nextAttemptAt?: string;
|
|
227
272
|
leaseExpiresAt?: string;
|
|
228
273
|
workflowId?: string;
|
|
@@ -233,6 +278,7 @@ export interface WorkflowWorkItem {
|
|
|
233
278
|
updatedAt: string;
|
|
234
279
|
}
|
|
235
280
|
export interface UpsertWorkflowWorkItemInput {
|
|
281
|
+
id?: string;
|
|
236
282
|
routeKey: string;
|
|
237
283
|
idempotencyKey: string;
|
|
238
284
|
invocationId: string;
|
|
@@ -339,19 +385,44 @@ export interface WorkflowStepRun {
|
|
|
339
385
|
createdAt: string;
|
|
340
386
|
updatedAt: string;
|
|
341
387
|
}
|
|
342
|
-
export
|
|
388
|
+
export type WorkflowLifecycleEventType = "created" | "workflow_archived" | "todos_workflow_pointers_synced" | "todos_workflow_pointers_sync_failed" | "step_started" | "step_progress" | "recovered" | "step_pending" | "step_running" | "step_succeeded" | "step_failed" | "step_timed_out" | "step_skipped" | "step_cancelled" | "succeeded" | "failed" | "timed_out" | "cancelled";
|
|
389
|
+
export interface WorkflowEventBase {
|
|
343
390
|
id: string;
|
|
344
391
|
workflowRunId: string;
|
|
345
392
|
sequence: number;
|
|
346
|
-
eventType: string;
|
|
347
393
|
stepId?: string;
|
|
348
394
|
payload?: Record<string, unknown>;
|
|
349
395
|
createdAt: string;
|
|
350
396
|
}
|
|
397
|
+
/** Raw persisted event shape used inside storage adapters before public validation. */
|
|
398
|
+
export interface StoredWorkflowEvent extends WorkflowEventBase {
|
|
399
|
+
eventType: string;
|
|
400
|
+
}
|
|
401
|
+
export interface GenericWorkflowEvent extends WorkflowEventBase {
|
|
402
|
+
eventType: WorkflowLifecycleEventType;
|
|
403
|
+
}
|
|
404
|
+
export interface AgentSessionContractWorkflowEvent extends Omit<WorkflowEventBase, "stepId" | "payload"> {
|
|
405
|
+
eventType: "agent_session_contract";
|
|
406
|
+
stepId: string;
|
|
407
|
+
payload: AgentSessionContract;
|
|
408
|
+
}
|
|
409
|
+
/**
|
|
410
|
+
* Backwards-compatible public shape for historical or mixed-version custom
|
|
411
|
+
* events. `eventKind` makes the generic branch unambiguous without weakening
|
|
412
|
+
* the schemas of server-owned lifecycle and agent-session-contract events.
|
|
413
|
+
*/
|
|
414
|
+
export interface CustomWorkflowEvent extends WorkflowEventBase {
|
|
415
|
+
eventKind: "custom";
|
|
416
|
+
eventType: string;
|
|
417
|
+
}
|
|
418
|
+
export type WorkflowEvent = AgentSessionContractWorkflowEvent | GenericWorkflowEvent;
|
|
419
|
+
export type PublicWorkflowEvent = WorkflowEvent | CustomWorkflowEvent;
|
|
351
420
|
export interface Loop {
|
|
352
421
|
id: string;
|
|
353
422
|
name: string;
|
|
354
423
|
description?: string;
|
|
424
|
+
/** Persisted loop labels. Legacy in-memory fixtures may omit this; stores normalize it to an empty array. */
|
|
425
|
+
labels?: string[];
|
|
355
426
|
status: LoopStatus;
|
|
356
427
|
archivedAt?: string;
|
|
357
428
|
archivedFromStatus?: LoopStatus;
|
|
@@ -446,6 +517,7 @@ export interface WriteRunReceiptInput {
|
|
|
446
517
|
export interface CreateLoopInput {
|
|
447
518
|
name: string;
|
|
448
519
|
description?: string;
|
|
520
|
+
labels?: string[];
|
|
449
521
|
schedule: ScheduleSpec;
|
|
450
522
|
target: LoopTargetInput;
|
|
451
523
|
goal?: GoalSpec;
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# Automation Runtime Design
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Loops can execute workflow work that external automation systems have
|
|
4
4
|
already materialized, but it must not become the automation product surface.
|
|
5
5
|
`@hasna/automations` and `@hasna/actions` own automation specs, trigger
|
|
6
6
|
materialization, queue state, approvals, DLQ/replay, idempotency, and audit
|
|
7
|
-
evidence.
|
|
7
|
+
evidence. Loops owns workflow invocation, admission, execution, run
|
|
8
8
|
manifests, and provider routing once work is explicitly handed off.
|
|
9
9
|
|
|
10
10
|
For operator-facing ownership tables, handoff path examples, and anti-patterns,
|
|
@@ -12,7 +12,7 @@ see [Runtime Boundary](./RUNTIME_BOUNDARY.md).
|
|
|
12
12
|
|
|
13
13
|
## Planned Workflow Upsert SDK
|
|
14
14
|
|
|
15
|
-
External compilers should not write
|
|
15
|
+
External compilers should not write Loops SQLite rows directly. The stable
|
|
16
16
|
contract should be an idempotent CLI/SDK upsert that accepts a fully rendered
|
|
17
17
|
one-shot workflow loop request and returns durable refs.
|
|
18
18
|
|
|
@@ -63,7 +63,7 @@ type WorkflowUpsertResult = {
|
|
|
63
63
|
Required semantics:
|
|
64
64
|
|
|
65
65
|
- `mode="dry-run"` validates, canonicalizes, hashes, and returns the same JSON
|
|
66
|
-
shape without mutating
|
|
66
|
+
shape without mutating Loops state.
|
|
67
67
|
- `mode="preflight"` additionally checks provider binaries, machine routing,
|
|
68
68
|
accounts/auth profiles, prompt files, and workflow target compatibility before
|
|
69
69
|
commit.
|
|
@@ -81,8 +81,8 @@ Required semantics:
|
|
|
81
81
|
|
|
82
82
|
`@hasna/actions` now exposes typed action contracts: `ActionManifest`,
|
|
83
83
|
`ActionInvocation`, `ActionRun`, `ActionRunStatus`, `ActionQueueStatus`,
|
|
84
|
-
`ActionAuditEvent`, `EvidenceRef`, and `ActionDeadLetter`. A future
|
|
85
|
-
action binding must reuse those contracts instead of adding
|
|
84
|
+
`ActionAuditEvent`, `EvidenceRef`, and `ActionDeadLetter`. A future Loops
|
|
85
|
+
action binding must reuse those contracts instead of adding a Loops-owned
|
|
86
86
|
action status, action queue, or idempotency dialect.
|
|
87
87
|
|
|
88
88
|
The binding should be a workflow-admission descriptor, not a direct executor in
|
|
@@ -124,14 +124,14 @@ The planned implementation path is:
|
|
|
124
124
|
support, idempotency requirement, guardrails, audit fields, evidence fields,
|
|
125
125
|
and rollback policy using `@hasna/actions`.
|
|
126
126
|
2. The action owner passes only a normalized `ActionTargetBindingRequest` or
|
|
127
|
-
`WorkflowUpsertRequest` to
|
|
127
|
+
`WorkflowUpsertRequest` to Loops. Large inputs, private prompts, secret
|
|
128
128
|
values, and mutable action state stay in the action store or a referenced
|
|
129
129
|
artifact.
|
|
130
|
-
3.
|
|
130
|
+
3. Loops admits a one-shot workflow loop through the planned upsert
|
|
131
131
|
contract, records workflow/run/manifests refs, and reports those refs back to
|
|
132
132
|
the action owner.
|
|
133
133
|
4. The action owner records `ActionAuditEvent` and `EvidenceRef` entries that
|
|
134
|
-
include the returned
|
|
134
|
+
include the returned Loops refs. Loops may store source refs for
|
|
135
135
|
lookup, but `@hasna/actions` remains the source of truth for action runs and
|
|
136
136
|
queue state.
|
|
137
137
|
|
|
@@ -160,36 +160,36 @@ const request: WorkflowUpsertRequest = {
|
|
|
160
160
|
};
|
|
161
161
|
```
|
|
162
162
|
|
|
163
|
-
`specHash` must be computed from the canonical
|
|
163
|
+
`specHash` must be computed from the canonical Loops workflow request plus
|
|
164
164
|
the stable action contract identity: action id, manifest version, idempotency
|
|
165
165
|
key or invocation id, selected executor binding kind/ref, route policy, and
|
|
166
166
|
execution policy. It must not include raw secret values, raw credentials, or
|
|
167
167
|
unredacted prompt bodies. If `@hasna/actions` later publishes its own manifest
|
|
168
|
-
hash,
|
|
168
|
+
hash, Loops should include that hash instead of re-hashing action internals.
|
|
169
169
|
|
|
170
170
|
`WorkflowUpsertResult` keeps its existing semantics:
|
|
171
171
|
|
|
172
172
|
- `idempotencyKey` is the action-owned idempotency key selected above.
|
|
173
|
-
- `specHash` identifies the exact
|
|
173
|
+
- `specHash` identifies the exact Loops workflow admission request and
|
|
174
174
|
action contract identity that produced it.
|
|
175
175
|
- `refs.workflowId`, `refs.loopId`, `refs.invocationId`, `refs.workItemId`,
|
|
176
|
-
`refs.runId`, and `refs.manifestPath` are
|
|
176
|
+
`refs.runId`, and `refs.manifestPath` are Loops refs that the action
|
|
177
177
|
owner can store as `EvidenceRef` or audit event data.
|
|
178
|
-
- `action="created" | "updated" | "reused" | "rejected"` describes
|
|
178
|
+
- `action="created" | "updated" | "reused" | "rejected"` describes Loops
|
|
179
179
|
materialization only. It is not an action run status.
|
|
180
180
|
- Action run fields such as `status`, `dedupedFromRunId`, `evidence`, `events`,
|
|
181
181
|
and `error` remain on `ActionRun`.
|
|
182
182
|
|
|
183
183
|
### Status Translation
|
|
184
184
|
|
|
185
|
-
|
|
185
|
+
Loops should translate its workflow and work-item state into action-owned
|
|
186
186
|
status updates without persisting a new action status enum.
|
|
187
187
|
|
|
188
188
|
Recommended initial translation:
|
|
189
189
|
|
|
190
190
|
- An accepted dry-run/preflight maps to an `ActionRun` that remains
|
|
191
191
|
`planned`, `previewed`, or `awaiting_approval` according to
|
|
192
|
-
`@hasna/actions`;
|
|
192
|
+
`@hasna/actions`; Loops only returns preflight details.
|
|
193
193
|
- A scheduled workflow maps to action queue status `queued` or
|
|
194
194
|
`waiting_approval`, depending on action-owned approval gates.
|
|
195
195
|
- A claimed/running workflow maps to action queue status `claimed` and action
|
|
@@ -202,8 +202,8 @@ Recommended initial translation:
|
|
|
202
202
|
- Operator cancellation maps to action queue status `cancelled` and action run
|
|
203
203
|
status `cancelled`.
|
|
204
204
|
|
|
205
|
-
These mappings are adapter behavior.
|
|
206
|
-
statuses for
|
|
205
|
+
These mappings are adapter behavior. Loops should expose its own workflow
|
|
206
|
+
statuses for Loops APIs and should call or return enough refs for
|
|
207
207
|
`@hasna/actions` to update `ActionRun`, `ActionQueueStatus`, audit events, and
|
|
208
208
|
dead-letter records.
|
|
209
209
|
|
|
@@ -212,19 +212,19 @@ dead-letter records.
|
|
|
212
212
|
Failure handling must flow through action-owned APIs:
|
|
213
213
|
|
|
214
214
|
- Provider, worktree, account, policy, and command failures are classified by
|
|
215
|
-
|
|
215
|
+
Loops and returned as redacted workflow error evidence.
|
|
216
216
|
- The action owner turns those failures into `ActionError`,
|
|
217
217
|
`ActionDeadLetter`, `ActionAuditEvent`, and `EvidenceRef` records.
|
|
218
218
|
- Replay starts from an action-owned replay decision and a new action-owned
|
|
219
|
-
idempotency key.
|
|
219
|
+
idempotency key. Loops then receives a new upsert request whose
|
|
220
220
|
`source.dedupeKey` and `idempotencyKey` are the replay keys.
|
|
221
|
-
-
|
|
221
|
+
- Loops DLQ commands may mirror or inspect linked workflow failures, but
|
|
222
222
|
they must not fabricate action queue entries or mark an action replayable on
|
|
223
223
|
their own.
|
|
224
224
|
|
|
225
225
|
### Mode And Dispatch Examples
|
|
226
226
|
|
|
227
|
-
Dry-run without
|
|
227
|
+
Dry-run without Loops mutation:
|
|
228
228
|
|
|
229
229
|
```bash
|
|
230
230
|
actions run deploy.manifest.json \
|
|
@@ -282,9 +282,9 @@ loops workflows upsert-one-shot deploy-workflow.json \
|
|
|
282
282
|
|
|
283
283
|
### Non-Goals
|
|
284
284
|
|
|
285
|
-
- Do not let `@hasna/actions` write
|
|
286
|
-
- Do not duplicate the action queue in
|
|
287
|
-
- Do not materialize triggers in
|
|
285
|
+
- Do not let `@hasna/actions` write Loops SQLite/Postgres rows directly.
|
|
286
|
+
- Do not duplicate the action queue in Loops.
|
|
287
|
+
- Do not materialize triggers in Loops; action and automation owners decide
|
|
288
288
|
when a concrete invocation exists.
|
|
289
289
|
- Do not store secret values, unredacted prompts, credentials, or private
|
|
290
290
|
action input payloads in workflow specs, prompts, manifests, task comments,
|
|
@@ -421,7 +421,7 @@ Required check domains:
|
|
|
421
421
|
result.
|
|
422
422
|
- `connector`: ask the owning action/connector system whether the connector
|
|
423
423
|
credential handle exists, is scoped for the source action, and is usable; do
|
|
424
|
-
not make
|
|
424
|
+
not make Loops the connector credential store.
|
|
425
425
|
- `permissions`: reject unsafe `permissionMode="bypass"`, unsafe
|
|
426
426
|
`configIsolation="none"`, mutable command targets without an explicit policy,
|
|
427
427
|
or missing approval evidence when strict automation requires it.
|
|
@@ -463,7 +463,7 @@ worktrees for repo mutation, missing prompt files, and machine route mismatch.
|
|
|
463
463
|
## Planned Compact Evidence API
|
|
464
464
|
|
|
465
465
|
Every public surface should expose the same compact evidence shape so callers
|
|
466
|
-
can hand results across `@hasna/actions`, `@hasna/automations`,
|
|
466
|
+
can hand results across `@hasna/actions`, `@hasna/automations`, Loops route
|
|
467
467
|
drains, MCP tools, and CLI scripts without scraping SQLite or leaking secrets.
|
|
468
468
|
|
|
469
469
|
```ts
|
|
@@ -515,7 +515,7 @@ Persistence and output rules:
|
|
|
515
515
|
work item, and owning external action/automation refs through public APIs.
|
|
516
516
|
- Failed preflight evidence should be persistable when a commit attempt is
|
|
517
517
|
rejected, but pure `dry-run`/`preflight` calls should return it directly
|
|
518
|
-
without creating
|
|
518
|
+
without creating Loops database rows.
|
|
519
519
|
- Redaction happens before persistence and before API return. Tests should
|
|
520
520
|
exercise both flat secret-looking text and nested JSON evidence values.
|
|
521
521
|
|
|
@@ -548,7 +548,7 @@ secret-safe without depending on ambient shell state.
|
|
|
548
548
|
`execution.mode="strict"` means:
|
|
549
549
|
|
|
550
550
|
- Do not inherit full `process.env`. Start from a minimal runtime environment:
|
|
551
|
-
`PATH`, `HOME`, `TMPDIR`, locale keys,
|
|
551
|
+
`PATH`, `HOME`, `TMPDIR`, locale keys, Loops metadata keys, and tool
|
|
552
552
|
config dirs explicitly produced by account/secret resolution.
|
|
553
553
|
- Pass only named `envAllowlist` values and `secretRefs`; never embed secret
|
|
554
554
|
values in workflow specs, prompts, metadata, manifests, or task comments.
|
|
@@ -600,7 +600,7 @@ enforcement surface is missing.
|
|
|
600
600
|
|
|
601
601
|
## Planned DLQ And Replay
|
|
602
602
|
|
|
603
|
-
|
|
603
|
+
Loops already models terminal route work items and has a manual
|
|
604
604
|
`routes requeue` command. The automation runtime contract needs a stricter DLQ
|
|
605
605
|
layer for exhausted side-effecting work, because external action systems need to
|
|
606
606
|
distinguish "failed attempt", "dead until operator action", and "resolved".
|
|
@@ -615,7 +615,7 @@ Route work items should keep these meanings:
|
|
|
615
615
|
as unsafe to retry without operator action.
|
|
616
616
|
- `cancelled`: an operator or owner intentionally stopped the work.
|
|
617
617
|
- `succeeded`: the work completed and remains deduped history.
|
|
618
|
-
- `resolved`: planned external DLQ state for `@hasna/actions`;
|
|
618
|
+
- `resolved`: planned external DLQ state for `@hasna/actions`; Loops can
|
|
619
619
|
mirror it as `dead_letter` plus a resolution event until a first-class status
|
|
620
620
|
is added.
|
|
621
621
|
|
|
@@ -648,7 +648,7 @@ provider code; it only records why the DLQ entry no longer needs execution.
|
|
|
648
648
|
- Replay of side-effecting work should default to `mode=preflight`; promotion
|
|
649
649
|
to `mode=commit` is a separate operator/API decision unless the owning
|
|
650
650
|
`@hasna/actions` policy marks the action safely replayable.
|
|
651
|
-
- If `@hasna/actions` owns the queue,
|
|
651
|
+
- If `@hasna/actions` owns the queue, Loops calls the action replay API and
|
|
652
652
|
stores the returned action/run refs instead of fabricating queue state.
|
|
653
653
|
|
|
654
654
|
### Dead-Letter Classification
|
package/docs/CUTOVER-RUNBOOK.md
CHANGED
|
@@ -5,16 +5,14 @@ complete.** Do not flip scheduled production execution away from local SQLite
|
|
|
5
5
|
until the runner and migration follow-ups below are green.
|
|
6
6
|
|
|
7
7
|
Deployment vocabulary: `self_hosted` is the Hasna-owned AWS/RDS control-plane
|
|
8
|
-
deployment. `cloud` is the future hosted SaaS contract for outside users.
|
|
9
|
-
vendored `@hasna/contracts` storage kit still calls direct Postgres storage
|
|
10
|
-
mode `cloud`; treat that as a storage-kit implementation term, not the
|
|
11
|
-
OpenLoops deployment mode.
|
|
8
|
+
deployment. `cloud` is the future hosted SaaS contract for outside users.
|
|
12
9
|
|
|
13
10
|
## What Shipped
|
|
14
11
|
|
|
15
12
|
- `loops-serve` HTTP control plane: RDS-direct Postgres, `GET /health`,
|
|
16
13
|
`/ready`, `/version`, `/openapi.json`, storage-backed `/v1` loop CRUD and run
|
|
17
|
-
listing, and runner
|
|
14
|
+
listing, and runner claim/lease heartbeat/finalize protocol routes. Durable
|
|
15
|
+
runner registration remains a cutover gate and is not advertised as a route.
|
|
18
16
|
- `@hasna/contracts` API-key auth on non-local `loops-serve` binds, backed by
|
|
19
17
|
the shared `api_keys` table and a signing secret.
|
|
20
18
|
- Full `PostgresLoopStorage` behind `LoopStorageContract`, plus
|
|
@@ -23,7 +21,7 @@ OpenLoops deployment mode.
|
|
|
23
21
|
and ensures the `api_keys` table.
|
|
24
22
|
- `@hasna/loops/sdk/http`, generated from `openapi/loops.json`.
|
|
25
23
|
- ARM64/Bun `Dockerfile`, local `docker-compose.yml`, `hasna.contract.json`,
|
|
26
|
-
and a generated `migrations/` mirror for review.
|
|
24
|
+
generated storage kit, and a generated `migrations/` mirror for review.
|
|
27
25
|
|
|
28
26
|
## Local Postgres Smoke For loops-serve
|
|
29
27
|
|
|
@@ -37,42 +35,171 @@ curl -fsS http://127.0.0.1:8787/ready
|
|
|
37
35
|
curl -fsS http://127.0.0.1:8787/openapi.json
|
|
38
36
|
```
|
|
39
37
|
|
|
40
|
-
`loops-serve`
|
|
41
|
-
|
|
42
|
-
`HASNA_API_SIGNING_KEY`, or `API_KEY_SIGNING_SECRET`; otherwise the service
|
|
43
|
-
fails closed before exposing `/v1`.
|
|
38
|
+
`loops-serve` always requires `HASNA_LOOPS_API_SIGNING_KEY`. There is no
|
|
39
|
+
loopback authentication bypass.
|
|
44
40
|
|
|
45
41
|
## AWS Self-Hosted Gates
|
|
46
42
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
43
|
+
Before step 1, satisfy and preserve evidence for these hard gates:
|
|
44
|
+
|
|
45
|
+
- For the shared `apps` to dedicated `loops` database transfer lane, use only
|
|
46
|
+
the selective logical transfer in `docs/SHARED-DATABASE-TRANSFER.md`. Do not
|
|
47
|
+
snapshot-restore the shared cluster or shared database into the dedicated
|
|
48
|
+
Loops target. The protected workflow may start only the fixed
|
|
49
|
+
`bun dist/serve/index.js shared-to-dedicated-transfer` ECS command, with DSNs
|
|
50
|
+
supplied by ECS task secrets.
|
|
51
|
+
- Use a dedicated Loops PostgreSQL cluster, not only a dedicated database.
|
|
52
|
+
PostgreSQL roles are cluster-global: inventory every database, role
|
|
53
|
+
membership, database owner, and `pg_shdepend` row for the four reserved
|
|
54
|
+
`open_loops_*` roles. The enforcement preflight fails if a reserved role owns
|
|
55
|
+
another database or has a cross-database dependency. No reserved role may be
|
|
56
|
+
`LOGIN`; the command fails closed instead of silently detaching a credential.
|
|
57
|
+
- Prove recovery before mutation. Confirm PITR is inside its retention window,
|
|
58
|
+
identify the exact pre-cutover recovery point, and complete an isolated
|
|
59
|
+
restore rehearsal from that point. Record the recovery point, restored
|
|
60
|
+
database identifier, verification command/results, elapsed restore time, and
|
|
61
|
+
cleanup result. Backup configuration without a successful restore is not
|
|
62
|
+
sufficient evidence.
|
|
63
|
+
- Enter a maintenance window before the first non-dry-run migration. Stop new
|
|
64
|
+
API writes, drain in-flight work, stop every service/runner that can write the
|
|
65
|
+
database, and verify no non-operator sessions or open write transactions
|
|
66
|
+
remain. Run exactly one migrator. The binary also takes a transaction-scoped
|
|
67
|
+
advisory lock and recomputes the ledger plan after acquiring it, but the lock
|
|
68
|
+
does not replace application quiescence.
|
|
69
|
+
|
|
70
|
+
1. Provision the four NOLOGIN database roles and the separate enforcement
|
|
71
|
+
login. Tenant enforcement must run through a
|
|
72
|
+
provider-level bootstrap administrator against a dedicated Loops
|
|
73
|
+
database owned by that bootstrap login (or use a true superuser). Do not run enforcement in a database shared with another
|
|
74
|
+
application: migration `0010` normalizes ACLs across every non-system
|
|
75
|
+
schema, table, sequence, and function in the current database. `CREATEROLE`, control of the
|
|
76
|
+
`public` schema, and the ability to `SET ROLE` to `open_loops_owner` and
|
|
77
|
+
`open_loops_migrator` are minimum requirements. With PostgreSQL 16's default
|
|
78
|
+
`createrole_self_grant=''`, a non-superuser must use four pre-provisioned,
|
|
79
|
+
already-normalized Loops roles and must have only direct owner/migrator
|
|
80
|
+
memberships with `ADMIN FALSE`, `INHERIT TRUE`, and `SET TRUE`. It must not
|
|
81
|
+
be a member of the runtime or authenticator roles. No LOGIN role may inherit
|
|
82
|
+
runtime or authenticator yet; attach service credentials only after `0010`
|
|
83
|
+
succeeds. Creating the roles as the
|
|
84
|
+
non-superuser is not sufficient: PostgreSQL adds an implicit `ADMIN TRUE`,
|
|
85
|
+
`INHERIT FALSE`, `SET FALSE` creator row that only a superuser can revoke.
|
|
86
|
+
The command transactionally exercises and rolls back the exact membership,
|
|
87
|
+
`SET ROLE`, migration-ledger ownership/write, ACL, and service-login cleanup
|
|
88
|
+
operations before migration `0010`; static role flags are not accepted as
|
|
89
|
+
proof. Provider/bootstrap identities are never passed to `DROP OWNED`, and
|
|
90
|
+
unsafe LOGIN memberships fail closed instead of being silently detached.
|
|
91
|
+
After enforcement, the runtime and authenticator service logins must be
|
|
92
|
+
members only of their matching roles, granted with `ADMIN FALSE`, `INHERIT
|
|
93
|
+
TRUE`, and `SET TRUE`.
|
|
94
|
+
Never reuse the bootstrap DSN in `loops-serve`.
|
|
95
|
+
2. Prepare the tenant schema with the ECS one-shot task or an operator command:
|
|
96
|
+
`HASNA_LOOPS_MIGRATOR_DATABASE_URL=... loops-serve migrate --dry-run`, then
|
|
97
|
+
`HASNA_LOOPS_MIGRATOR_DATABASE_URL=... loops-serve migrate`.
|
|
98
|
+
3. In ECS, load the reviewed explicit ownership bundle with the fixed,
|
|
99
|
+
no-argument command
|
|
100
|
+
`HASNA_LOOPS_MIGRATOR_DATABASE_URL=... HASNA_LOOPS_BACKFILL_BUCKET=... AWS_REGION=... loops-serve tenant-backfill-s3`.
|
|
101
|
+
The task role must expose only a valid
|
|
102
|
+
`AWS_CONTAINER_CREDENTIALS_RELATIVE_URI`. The bucket must contain exactly one
|
|
103
|
+
object under `approved/`, named `approved/sha256-<64 lowercase hex>.json`,
|
|
104
|
+
and no more than 10 MiB. The command verifies the raw-byte digest before
|
|
105
|
+
parsing, loads transactionally, deletes the selected object on success or
|
|
106
|
+
failure, and treats deletion failure as fatal. Record its bounded digest and
|
|
107
|
+
counts log only. For a local operator rehearsal, the compatible file command
|
|
108
|
+
remains `HASNA_LOOPS_MIGRATOR_DATABASE_URL=... loops-serve tenant-backfill --input <bundle>`.
|
|
109
|
+
Before delivery, require named operator approval of tenant, principal,
|
|
110
|
+
membership, API-key, and row-owner counts; compute and record the bundle's
|
|
111
|
+
SHA-256; and transfer it only through the approved encrypted artifact path.
|
|
112
|
+
For local file delivery only, verify the hash and restrictive file
|
|
113
|
+
permissions before use, then remove the staged artifact after the
|
|
114
|
+
transaction and evidence capture complete. Record only the hash, counts,
|
|
115
|
+
approver, and bounded command result—never bundle contents or credentials.
|
|
116
|
+
4. Enforce tenant keys, composite foreign keys, and forced RLS with
|
|
117
|
+
`HASNA_LOOPS_MIGRATOR_DATABASE_URL=... loops-serve migrate --enforce-tenancy`.
|
|
118
|
+
After this succeeds, have provider automation attach the runtime and
|
|
119
|
+
authenticator logins to their matching roles; do not reuse the enforcement
|
|
120
|
+
login for either service.
|
|
121
|
+
Keep the write plane quiesced while validating migration-ledger ownership,
|
|
122
|
+
exact role memberships, forced RLS, runtime/authenticator connection safety,
|
|
123
|
+
and `/ready`. If any gate fails, keep the service stopped and use the
|
|
124
|
+
rehearsed recovery procedure; do not continue from a partially understood
|
|
125
|
+
state.
|
|
126
|
+
5. Build and deploy an image from this repository's pinned Bun base-image
|
|
127
|
+
digest. The runner image must pass the CI high/critical vulnerability scan
|
|
128
|
+
and must not replace the locked `@hasna/contracts` package with a vendored
|
|
129
|
+
overlay.
|
|
130
|
+
6. Configure the ECS service before traffic is shifted:
|
|
131
|
+
- desired count at least `2`;
|
|
132
|
+
- tasks spread across at least two private subnets/AZs;
|
|
133
|
+
- capacity provider strategy with an on-demand Fargate base of at least one
|
|
134
|
+
task before any Spot capacity;
|
|
135
|
+
- deployment circuit breaker with rollback enabled;
|
|
136
|
+
- ALB target group health check path `/ready`;
|
|
137
|
+
- CloudWatch log retention at least 30 days with KMS encryption;
|
|
138
|
+
- alarm actions wired to the approved incident notification target.
|
|
139
|
+
7. Store runtime values in Secrets Manager or the approved vault surface:
|
|
140
|
+
`HASNA_LOOPS_DATABASE_URL`, `HASNA_LOOPS_AUTH_DATABASE_URL`,
|
|
141
|
+
`HASNA_LOOPS_MIGRATOR_DATABASE_URL`, and
|
|
142
|
+
`HASNA_LOOPS_API_SIGNING_KEY`. Enable rotation for the signing secret and
|
|
143
|
+
database credentials where the provider supports it. Do not put secret values
|
|
144
|
+
in task definitions, task comments, logs, or rollout evidence.
|
|
145
|
+
8. When provider-managed RDS credentials are used, run
|
|
146
|
+
`loops-serve db-credentials reconcile` from the in-cluster task role before
|
|
147
|
+
starting the service. Required inputs are secret ARNs and expected RDS
|
|
148
|
+
metadata only, not secret values. The command reads the RDS-managed master
|
|
149
|
+
secret through AWS Secrets Manager, writes app DSNs through `AWSPENDING`,
|
|
150
|
+
changes each login password transactionally, verifies a fresh
|
|
151
|
+
`sslmode=verify-full` connection, and promotes `AWSCURRENT` only after the
|
|
152
|
+
fresh connection succeeds. It logs only bounded status metadata. Before
|
|
153
|
+
migration `0010_tenant_enforce`, runtime/authenticator logins remain detached;
|
|
154
|
+
after `0010`, they attach only to their matching NOLOGIN roles.
|
|
155
|
+
9. Wire minimum alarms before cutover: ALB unhealthy hosts, ALB 5xx, ALB target
|
|
156
|
+
latency, ECS running count below desired count, ECS task exits, RDS CPU,
|
|
157
|
+
RDS connections, RDS free storage, and log error-rate/auth-anomaly signals.
|
|
158
|
+
10. Start `loops-serve` with `HASNA_LOOPS_STORAGE_MODE=self_hosted`, separate
|
|
159
|
+
`HASNA_LOOPS_DATABASE_URL` and `HASNA_LOOPS_AUTH_DATABASE_URL` logins, and the API signing secret from the approved
|
|
160
|
+
vault item. The signing key must be at least 16 bytes. Do not log or copy the
|
|
161
|
+
secret value into task evidence.
|
|
162
|
+
11. Verify `/health`, `/ready`, `/version`, and `/openapi.json`.
|
|
163
|
+
12. Verify an authenticated `/v1` read/write smoke against a throwaway loop and
|
|
164
|
+
a claim/finalize smoke if a runner API URL is configured.
|
|
165
|
+
13. Record package version, git SHA, image tag and digest, database migration
|
|
166
|
+
plan/result, evidence that the target database is dedicated to Loops,
|
|
167
|
+
redacted API URL, health/readiness responses, capacity-provider strategy,
|
|
168
|
+
desired/running task counts, alarm action ARNs/names, log retention/KMS
|
|
169
|
+
identifiers, rotation status, and rollback handle.
|
|
58
170
|
|
|
59
171
|
## Still Pending Before Daemon Cutover
|
|
60
172
|
|
|
61
173
|
- Long-running `loops-runner` daemon mode with backoff, fleet observability, and
|
|
62
174
|
durable machine registration records.
|
|
63
|
-
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
175
|
+
- Id-preserving self-hosted import coverage for run history, workflow history,
|
|
176
|
+
work items, goals, and audit rows.
|
|
177
|
+
- A full-history no-loss migration path from local SQLite into the self-hosted
|
|
178
|
+
control plane. Current `loops export`/`loops import` and
|
|
179
|
+
`loops self-hosted push --apply` cover workflow specs and loop definitions
|
|
180
|
+
with safe paused/archived defaults; they intentionally block unsupported live
|
|
181
|
+
history.
|
|
69
182
|
- Hosted SaaS integration outside this public package.
|
|
70
183
|
|
|
71
184
|
## Rollback
|
|
72
185
|
|
|
73
|
-
|
|
74
|
-
or
|
|
75
|
-
|
|
76
|
-
|
|
186
|
+
Before migration `0010_tenant_enforce` is applied, an image-only rollback may
|
|
187
|
+
redeploy the previous digest or shift the ALB target group to the previous
|
|
188
|
+
service revision, provided its migration checksum contract still matches the
|
|
189
|
+
database. After `0010` succeeds, do not point a previous image at the enforced
|
|
190
|
+
database: the `c506685e` base carries the superseded `0010` checksum and the
|
|
191
|
+
published `0.4.28` image has no `0008`-`0010`, so neither can pass readiness
|
|
192
|
+
against this schema.
|
|
193
|
+
|
|
194
|
+
Post-`0010`, either roll forward with a schema-compatible image, or restore the
|
|
195
|
+
rehearsed pre-cutover PITR point to a separate database target. For the restore
|
|
196
|
+
path, repoint the previous service revision's runtime/auth credentials and
|
|
197
|
+
endpoint to that separate target, verify the restored migration ledger and
|
|
198
|
+
expected row counts, then prove the previous image's `/ready`, authenticated
|
|
199
|
+
loop CRUD, runner claim/finalize, and
|
|
200
|
+
`loops self-hosted push --dry-run --no-runs`. Never attempt an in-place reverse
|
|
201
|
+
migration or overwrite the enforced database during rollback.
|
|
202
|
+
Local scheduled execution remains on SQLite unless operators explicitly
|
|
203
|
+
configure a runner/control-plane cutover, so removing `HASNA_LOOPS_API_URL` and
|
|
77
204
|
`HASNA_LOOPS_DATABASE_URL` returns the standalone CLI/daemon perspective to
|
|
78
205
|
`local`.
|