@knpkv/herdr-work 0.5.3 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +103 -0
- package/dist/display-state.d.ts +10 -0
- package/dist/display-state.d.ts.map +1 -0
- package/dist/display-state.js +5 -0
- package/dist/display-state.js.map +1 -0
- package/dist/errors.d.ts +21 -0
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +12 -0
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/internal/legacy-lane-claim.d.ts +74 -0
- package/dist/internal/legacy-lane-claim.d.ts.map +1 -0
- package/dist/internal/legacy-lane-claim.js +81 -0
- package/dist/internal/legacy-lane-claim.js.map +1 -0
- package/dist/internal/session-index.d.ts +18 -0
- package/dist/internal/session-index.d.ts.map +1 -0
- package/dist/internal/session-index.js +23 -0
- package/dist/internal/session-index.js.map +1 -0
- package/dist/internal/terminal-reconcile.d.ts +31 -0
- package/dist/internal/terminal-reconcile.d.ts.map +1 -0
- package/dist/internal/terminal-reconcile.js +89 -0
- package/dist/internal/terminal-reconcile.js.map +1 -0
- package/dist/model.d.ts +939 -29
- package/dist/model.d.ts.map +1 -1
- package/dist/model.js +256 -7
- package/dist/model.js.map +1 -1
- package/dist/observed.d.ts +77 -0
- package/dist/observed.d.ts.map +1 -0
- package/dist/observed.js +240 -0
- package/dist/observed.js.map +1 -0
- package/dist/projection.d.ts +192 -12
- package/dist/projection.d.ts.map +1 -1
- package/dist/service.d.ts +121 -992
- package/dist/service.d.ts.map +1 -1
- package/dist/service.js +13 -3
- package/dist/service.js.map +1 -1
- package/dist/sql.d.ts.map +1 -1
- package/dist/sql.js +76 -3
- package/dist/sql.js.map +1 -1
- package/dist/store.d.ts +193 -11
- package/dist/store.d.ts.map +1 -1
- package/dist/store.js +768 -37
- package/dist/store.js.map +1 -1
- package/dist/view.d.ts.map +1 -1
- package/dist/view.js +169 -103
- package/dist/view.js.map +1 -1
- package/dist/work-triage.d.ts +72 -0
- package/dist/work-triage.d.ts.map +1 -0
- package/dist/work-triage.js +114 -0
- package/dist/work-triage.js.map +1 -0
- package/package.json +3 -3
- package/src/styles.css +229 -237
package/dist/store.js
CHANGED
|
@@ -1,16 +1,20 @@
|
|
|
1
|
-
import { agentConnectTarget, fleetResponseBodyMaxBytes, workReassignActivitySummary } from "@knpkv/herdr-fleet";
|
|
1
|
+
import { agentConnectTarget, fleetResponseBodyMaxBytes, workAbandonActivitySummary, workReassignActivitySummary } from "@knpkv/herdr-fleet";
|
|
2
2
|
import { openPrivateSqlite } from "@knpkv/herdr-fleet/sqlite";
|
|
3
|
-
import { Clock, Crypto, Effect, Equal, Option, Schema } from "effect";
|
|
3
|
+
import { Clock, Crypto, Effect, Equal, Option, Result, Schema } from "effect";
|
|
4
4
|
import { Hex } from "effect/encoding";
|
|
5
5
|
import { makeWorkAgentBinding } from "./agent-binding.js";
|
|
6
|
-
import { WorkAdmissionConflictError, WorkAgentBindingAuthorityError, WorkAgentBindingConflictError, WorkCheckpointConflictError, WorkCoordinatorHandoffConflictError, WorkDecisionAuthorityConflictError, WorkDecisionHandoffConflictError, WorkDecisionRevisionConflictError, WorkGoalAgentTargetConflictError, WorkGoalBindingRequiresAgentError, WorkGoalOwnerMismatchError, WorkGoalReassignmentConflictError, WorkGoalRevisionConflictError, WorkLaneClaimConflictError, WorkLaneGoalConflictError, WorkLaneOperationConflictError, WorkProjectionError, WorkPullRequestLinkError, WorkStoreError, WorkTransactionConflictError } from "./errors.js";
|
|
6
|
+
import { WorkAdmissionConflictError, WorkAgentBindingAuthorityError, WorkAgentBindingConflictError, WorkCheckpointConflictError, WorkCoordinatorHandoffConflictError, WorkDecisionAuthorityConflictError, WorkDecisionHandoffConflictError, WorkDecisionRevisionConflictError, WorkGoalAbandonmentConflictError, WorkGoalAgentTargetConflictError, WorkGoalBindingRequiresAgentError, WorkGoalLaneActiveError, WorkGoalOwnerMismatchError, WorkGoalReassignmentConflictError, WorkGoalRevisionConflictError, WorkGoalTerminalError, WorkLaneClaimConflictError, WorkLaneGoalConflictError, WorkLaneOperationConflictError, WorkProjectionError, WorkPullRequestLinkError, WorkStoreError, WorkTransactionConflictError } from "./errors.js";
|
|
7
7
|
import { validateGoalFamilyHistory } from "./goal-family.js";
|
|
8
8
|
import { agentBindingAdmissionError, workAgentBindingLaneOperationMaxBytes, workAgentBindingLaneOperationMaxRecords, workAgentBindingMaximumSnapshotBytes, workAgentBindingSnapshotEnvelopeMaxBytes, workMaximumSnapshotBytesForHistory } from "./internal/agent-binding-admission.js";
|
|
9
9
|
import { AgentBindingGoalEventRow, AgentBindingLaneOperationRow, agentBindingReadbackError, AgentBindingRow, decodeAgentBindingGoalEvent, decodeAgentBindingRow, laneOperationReadbackError } from "./internal/agent-binding-readback.js";
|
|
10
10
|
import { CoordinatorCommandRow, CoordinatorLifecycleDispatchRow, CoordinatorLifecycleEventRow, coordinatorLifecycleRunningAt, CoordinatorRouteDiscriminatorRow, coordinatorRouteRequiresWorkLink, requireCoordinatorFailedLunaAuthority, requireCoordinatorLifecycleAuthority, requireCoordinatorRouteAuthority, requireCoordinatorRouteBinding } from "./internal/coordinator-authority.js";
|
|
11
11
|
import { currentDecisionHandoffEquivalent, CurrentMetadataWorkLink, decodePreviousDecisionHandoff, previousDecisionHandoffEquivalent, PreviousWorkDecisionHandoff, upgradePreviousDecisionHandoff, workDispatchLineageContainedBy, workDispatchLineageEquivalent } from "./internal/decision-handoff-migration.js";
|
|
12
12
|
import { workHistoryError } from "./internal/history-validation.js";
|
|
13
|
-
import {
|
|
13
|
+
import { LaneOperationLedgerRow, LaneOperationTotalsRow, planLegacyLaneOperations, resolveLegacyLaneClaim } from "./internal/legacy-lane-claim.js";
|
|
14
|
+
import { enforcesOneHandoffPerSession, SessionIndexDefinition, sessionIndexDefinitionQuery } from "./internal/session-index.js";
|
|
15
|
+
import { goalActivityLimit, terminalCandidates, terminalCheckpoint, withoutBlockers } from "./internal/terminal-reconcile.js";
|
|
16
|
+
import { admissionEvidence, isTerminalWorkState, WorkAdmissionTarget, WorkAgentBinding, WorkAgentBindingRequest, WorkCoordinatorSessionId, WorkDecisionHandoff, WorkDispatchHandoff, WorkExistingGoalRecovery, WorkGoal, WorkGoalAbandoned, WorkGoalAbandonment, WorkGoalCheckpoint, WorkGoalId, WorkGoalReassigned, WorkGoalReassignment, workHistoryMaxEvents, WorkLaneClaim, WorkLaneClaimed, WorkObservationEnvelope, WorkObservationId, workObservationMaxSkewMillis, WorkObservationSubject, WorkObservedAdmission, WorkObservedFact, workObservedFactMaxBytes, workObservedFactMaxRecords, WorkObservedFailure, WorkProspectiveAdmission, WorkPullRequestLink, WorkReconcileOptions, workReconcilerHeadroom, WorkRecoveryTarget, workSnapshotMaxGoals } from "./model.js";
|
|
17
|
+
import { asciiLower, canonicalSubject, observationSubject, reconcilerEventPrefix } from "./observed.js";
|
|
14
18
|
const StoredEventRow = Schema.Struct({ record: Schema.String });
|
|
15
19
|
const StoredEventRows = Schema.Array(StoredEventRow);
|
|
16
20
|
const StoredEventWithTransactionRow = Schema.Struct({
|
|
@@ -138,6 +142,31 @@ const legacyDecisionEquivalent = Schema.toEquivalence(LegacyDecisionRecord);
|
|
|
138
142
|
const TransactionId = Schema.String.check(Schema.isNonEmpty(), Schema.isMaxLength(256), Schema.isPattern(/^(?:[^\uD800-\uDFFF]|[\uD800-\uDBFF][\uDC00-\uDFFF])*$/));
|
|
139
143
|
const storeError = (operation) => (cause) => new WorkStoreError({ cause, operation });
|
|
140
144
|
const fromPrivateDatabaseError = (error) => new WorkStoreError({ cause: error.cause, operation: error.operation });
|
|
145
|
+
/**
|
|
146
|
+
* Files written before the session index was unique carry a plain index under
|
|
147
|
+
* the same name, which `CREATE UNIQUE INDEX IF NOT EXISTS` leaves in place.
|
|
148
|
+
* Replace it, failing closed when two handoffs already share a session. Runs
|
|
149
|
+
* inside `WorkStore.open`'s schema transaction, so a failure leaves the plain
|
|
150
|
+
* index in place.
|
|
151
|
+
*/
|
|
152
|
+
const requireUniqueSessionIndex = (database) => {
|
|
153
|
+
const sessionIndex = Schema.decodeUnknownSync(Schema.Array(SessionIndexDefinition))(database.prepare(sessionIndexDefinitionQuery).all())[0];
|
|
154
|
+
if (sessionIndex === undefined || enforcesOneHandoffPerSession(sessionIndex))
|
|
155
|
+
return;
|
|
156
|
+
// NULL sessions never collide under a UNIQUE index, so they are not duplicates.
|
|
157
|
+
const duplicateGroups = `SELECT session_id AS sessionId, COUNT(*) AS rows, group_concat(handoff_id) AS handoffIds
|
|
158
|
+
FROM work_decision_handoffs WHERE session_id IS NOT NULL
|
|
159
|
+
GROUP BY session_id HAVING COUNT(*) > 1`;
|
|
160
|
+
const duplicates = database.prepare(`${duplicateGroups} LIMIT 10`).all();
|
|
161
|
+
if (duplicates.length > 0) {
|
|
162
|
+
const total = database.prepare(`SELECT COUNT(*) AS sessions FROM (${duplicateGroups})`).get();
|
|
163
|
+
throw new WorkStoreError({ cause: { duplicates, total }, operation: "open.migrate.session-index" });
|
|
164
|
+
}
|
|
165
|
+
database.exec(`
|
|
166
|
+
DROP INDEX work_decision_handoffs_session;
|
|
167
|
+
CREATE UNIQUE INDEX work_decision_handoffs_session ON work_decision_handoffs (session_id);
|
|
168
|
+
`);
|
|
169
|
+
};
|
|
141
170
|
/**
|
|
142
171
|
* Upgrades pre-session Work tables in place. Runs inside `WorkStore.open`'s
|
|
143
172
|
* schema transaction, so a failure here or in a later schema step leaves the
|
|
@@ -371,6 +400,8 @@ const migrateLegacyAuthorityTables = (database) => {
|
|
|
371
400
|
});
|
|
372
401
|
})
|
|
373
402
|
: [];
|
|
403
|
+
// Lanes the running bindings of legacy handoffs recorded; see resolveLegacyLaneClaim.
|
|
404
|
+
const bindingLanes = [];
|
|
374
405
|
const decisions = decisionColumns.length > 0 && !decisionColumns.includes("session_id")
|
|
375
406
|
? Schema.decodeUnknownSync(Schema.Array(LegacyDecisionStoredRow))(database.prepare("SELECT handoff_id AS handoffId, record FROM work_decision_handoffs").all()).map((row) => {
|
|
376
407
|
const legacy = Schema.decodeUnknownSync(LegacyDecisionRecord)(JSON.parse(row.record));
|
|
@@ -454,6 +485,7 @@ const migrateLegacyAuthorityTables = (database) => {
|
|
|
454
485
|
operation: "open.migrate.legacy-handoff-lane-revision"
|
|
455
486
|
});
|
|
456
487
|
}
|
|
488
|
+
bindingLanes.push(bindingDecision.binding.lane);
|
|
457
489
|
requireBindingCompanions(bindingDecision.binding, "open.migrate.legacy-agent-binding");
|
|
458
490
|
requireLifecycleAuthority(bindingDecision.binding.request.dispatchRequestId, bindingDecision.binding.checkpoint.occurredAt, "open.migrate.legacy-agent-binding.lifecycle");
|
|
459
491
|
return Schema.decodeUnknownSync(WorkDecisionHandoff)({
|
|
@@ -468,13 +500,47 @@ const migrateLegacyAuthorityTables = (database) => {
|
|
|
468
500
|
});
|
|
469
501
|
})
|
|
470
502
|
: [];
|
|
503
|
+
const migratedClaims = lanes.map((lane) => resolveLegacyLaneClaim(lane, bindingLanes));
|
|
471
504
|
if (laneColumns.length > 0 && !laneColumns.includes("goal_id")) {
|
|
472
505
|
database.exec("ALTER TABLE work_lane_claims ADD COLUMN goal_id TEXT");
|
|
473
506
|
database.exec("ALTER TABLE work_lane_claims ADD COLUMN operation_id TEXT");
|
|
474
507
|
database.exec("ALTER TABLE work_lane_claims ADD COLUMN phase TEXT");
|
|
475
508
|
const update = database.prepare("UPDATE work_lane_claims SET goal_id = ?, operation_id = ?, phase = ?, record = ? WHERE lane_id = ?");
|
|
476
|
-
|
|
509
|
+
// An existing lane-operation ledger is not backfilled later, so record each
|
|
510
|
+
// migrated claim's operation here; the claim's readback requires it.
|
|
511
|
+
// Without a ledger, schema creation below creates it and backfills every
|
|
512
|
+
// claim, so the same bound applies to an empty ledger.
|
|
513
|
+
const hasLedger = tables.includes("work_lane_operations");
|
|
514
|
+
const operations = planLegacyLaneOperations(migratedClaims.map(({ lane }) => lane), new Map(hasLedger
|
|
515
|
+
? migratedClaims.flatMap(({ lane }) => {
|
|
516
|
+
const matches = database.prepare(`SELECT operation_id AS operationId, lane_id AS laneId, goal_id AS goalId, phase, revision, record
|
|
517
|
+
FROM work_lane_operations WHERE CAST(operation_id AS TEXT) = ?`).all(lane.operationId);
|
|
518
|
+
const rows = Schema.decodeUnknownSync(Schema.Array(LaneOperationLedgerRow))(matches);
|
|
519
|
+
// A key stored as a blob, or twice, is a collision even when its bytes match.
|
|
520
|
+
const found = rows.find(({ operationId }) => operationId !== lane.operationId) ?? rows[0];
|
|
521
|
+
return found === undefined ? [] : [[lane.operationId, found]];
|
|
522
|
+
})
|
|
523
|
+
: []), hasLedger
|
|
524
|
+
? Schema.decodeUnknownSync(LaneOperationTotalsRow)(database.prepare(`SELECT COUNT(*) AS count, COALESCE(SUM(
|
|
525
|
+
length(CAST(operation_id AS BLOB)) + length(CAST(record AS BLOB))), 0) AS bytes
|
|
526
|
+
FROM work_lane_operations`).get())
|
|
527
|
+
: { bytes: 0, count: 0 }, { bytes: workLaneOperationMaxBytes, records: workLaneOperationMaxRecords });
|
|
528
|
+
if (operations._tag === "collision") {
|
|
529
|
+
throw new WorkStoreError({ cause: operations, operation: "open.migrate.lane-operation-collision" });
|
|
530
|
+
}
|
|
531
|
+
if (operations._tag === "capacity") {
|
|
532
|
+
throw new WorkStoreError({ cause: operations, operation: "open.migrate.lane-operation-capacity" });
|
|
533
|
+
}
|
|
534
|
+
for (const { lane } of migratedClaims) {
|
|
477
535
|
update.run(lane.goalId, lane.operationId, lane.phase, JSON.stringify(lane), lane.laneId);
|
|
536
|
+
}
|
|
537
|
+
if (hasLedger && operations.inserts.length > 0) {
|
|
538
|
+
const recordOperation = database.prepare(`INSERT INTO work_lane_operations
|
|
539
|
+
(operation_id, lane_id, goal_id, phase, revision, record) VALUES (?, ?, ?, ?, ?, ?)`);
|
|
540
|
+
for (const lane of operations.inserts) {
|
|
541
|
+
recordOperation.run(lane.operationId, lane.laneId, lane.goalId, lane.phase, lane.revision, JSON.stringify(lane));
|
|
542
|
+
}
|
|
543
|
+
}
|
|
478
544
|
}
|
|
479
545
|
if (decisionColumns.length > 0 && !decisionColumns.includes("session_id")) {
|
|
480
546
|
database.exec("ALTER TABLE work_decision_handoffs ADD COLUMN session_id TEXT");
|
|
@@ -721,6 +787,16 @@ const migrateLegacyAuthorityTables = (database) => {
|
|
|
721
787
|
}
|
|
722
788
|
}
|
|
723
789
|
}
|
|
790
|
+
// Reported after capacity: lane CAS allows one binding per revision, so
|
|
791
|
+
// several mean the file is inconsistent rather than merely large.
|
|
792
|
+
const ambiguousClaims = migratedClaims.filter(({ _tag }) => _tag === "ambiguous");
|
|
793
|
+
if (ambiguousClaims.length > 0) {
|
|
794
|
+
throw new WorkStoreError({ cause: { claims: ambiguousClaims }, operation: "open.migrate.lane-binding-ambiguous" });
|
|
795
|
+
}
|
|
796
|
+
const mismatchedClaims = migratedClaims.filter(({ _tag }) => _tag === "mismatch");
|
|
797
|
+
if (mismatchedClaims.length > 0) {
|
|
798
|
+
throw new WorkStoreError({ cause: { claims: mismatchedClaims }, operation: "open.migrate.lane-binding-mismatch" });
|
|
799
|
+
}
|
|
724
800
|
if (tables.includes("orchestrator_dispatch_metadata") && coordinatorDispatchColumns.includes("is_routed")) {
|
|
725
801
|
const pageSize = 512;
|
|
726
802
|
let cursor = null;
|
|
@@ -794,6 +870,8 @@ const readDecisionLedgerTotals = (database) => Schema.decodeUnknownSync(Decision
|
|
|
794
870
|
FROM work_decision_totals WHERE singleton = 1`).get());
|
|
795
871
|
const readLaneOperationLedgerTotals = (database) => Schema.decodeUnknownSync(LaneOperationLedgerTotalsRow)(database.prepare(`SELECT operation_count AS operationCount, operation_bytes AS operationBytes
|
|
796
872
|
FROM work_lane_operation_totals WHERE singleton = 1`).get());
|
|
873
|
+
const ObservationIdRow = Schema.Struct({ observationId: Schema.String, confirmedAt: Schema.Number });
|
|
874
|
+
const FailureLastAtRow = Schema.Struct({ lastAt: Schema.Number });
|
|
797
875
|
const ReassignmentRow = Schema.Struct({ approvalJobId: Schema.String, goalId: Schema.String, record: Schema.String });
|
|
798
876
|
/** Applies a reassignment's agent choice; `keep` is only reached when the goal has no target. */
|
|
799
877
|
const withReassignedAgent = (goal, toAgent) => {
|
|
@@ -946,11 +1024,11 @@ const admissionState = (database, target) => {
|
|
|
946
1024
|
(claim.worktree === target.worktree && claim.laneId !== target.laneId)) ||
|
|
947
1025
|
authority.latest.some(({ request }) => request.worker.agentId === target.worker.agentId && request.laneId !== target.laneId) ||
|
|
948
1026
|
// A session id stays claimed by its lane even after a reassignment supersedes that binding.
|
|
949
|
-
bindings.some(({ request }) => (request
|
|
1027
|
+
bindings.some(({ request }) => (admissionEvidence(request)?.sessionId === target.sessionId && request.laneId !== target.laneId) ||
|
|
950
1028
|
(request.existingGoalRecovery?.sessionId === target.sessionId && request.laneId !== target.laneId)) || authority.ambiguousLanes.has(target.laneId);
|
|
951
1029
|
const exact = goal !== undefined && lane !== undefined && binding !== undefined &&
|
|
952
|
-
goal.goal.id === target.goalId && goal.goal.state
|
|
953
|
-
goal.goal.
|
|
1030
|
+
goal.goal.id === target.goalId && !isTerminalWorkState(goal.goal.state) &&
|
|
1031
|
+
goal.goal.goalFamily?.role === "canonical" &&
|
|
954
1032
|
goal.goal.goalFamily.canonicalGoalId === target.goalId &&
|
|
955
1033
|
Equal.equals(goal.goal.owner, target.owner) &&
|
|
956
1034
|
lane.goalId === target.goalId && lane.head === target.head &&
|
|
@@ -958,9 +1036,9 @@ const admissionState = (database, target) => {
|
|
|
958
1036
|
lane.phase !== "shipped" && Equal.equals(lane.owner, target.owner) &&
|
|
959
1037
|
binding.lane.laneId === target.laneId &&
|
|
960
1038
|
Equal.equals(binding.request.worker, target.worker) &&
|
|
961
|
-
(binding.request
|
|
962
|
-
binding.request
|
|
963
|
-
binding.request
|
|
1039
|
+
(admissionEvidence(binding.request) === undefined || (admissionEvidence(binding.request)?.sessionId === target.sessionId &&
|
|
1040
|
+
admissionEvidence(binding.request)?.workAssignment === target.expectedWork &&
|
|
1041
|
+
admissionEvidence(binding.request)?.baseHead === target.baseHead)) &&
|
|
964
1042
|
(binding.request.existingGoalRecovery === undefined ||
|
|
965
1043
|
(binding.request.existingGoalRecovery.sessionId === target.sessionId &&
|
|
966
1044
|
binding.request.existingGoalRecovery.workAssignment === target.expectedWork &&
|
|
@@ -987,7 +1065,7 @@ const admissionState = (database, target) => {
|
|
|
987
1065
|
entry.goalFamily?.canonicalGoalId === target.goalId ||
|
|
988
1066
|
entry.agentHierarchy?.agent.agentId === target.worker.agentId) || lanes.entries.some(({ claim }) => claim.laneId === target.laneId || claim.goalId === target.goalId ||
|
|
989
1067
|
claim.worktree === target.worktree) || authority.latest.some(({ request }) => request.worker.agentId === target.worker.agentId) ||
|
|
990
|
-
bindings.some(({ request }) => request
|
|
1068
|
+
bindings.some(({ request }) => admissionEvidence(request)?.sessionId === target.sessionId ||
|
|
991
1069
|
request.existingGoalRecovery?.sessionId === target.sessionId);
|
|
992
1070
|
if (conflicts) {
|
|
993
1071
|
return { _tag: "conflict", target, reason: "durable PR, goal, lane, worker, or worktree is already bound" };
|
|
@@ -1124,8 +1202,7 @@ const recoveryState = (database, target) => {
|
|
|
1124
1202
|
goal.connectTarget !== null ||
|
|
1125
1203
|
(goal.agentHierarchy !== undefined && goal.agentHierarchy !== null) ||
|
|
1126
1204
|
(goal.review?.url !== null && goal.review?.url !== undefined && goal.review.url !== target.reviewUrl) ||
|
|
1127
|
-
goal.state
|
|
1128
|
-
goal.state === "deployed")) {
|
|
1205
|
+
isTerminalWorkState(goal.state))) {
|
|
1129
1206
|
return conflict("canonical goal history is linked, terminal, or belongs to another owner");
|
|
1130
1207
|
}
|
|
1131
1208
|
const bindingRows = Schema.decodeUnknownSync(AgentBindingRows)(database
|
|
@@ -1158,7 +1235,7 @@ const recoveryState = (database, target) => {
|
|
|
1158
1235
|
operations.claims.some(({ goalId, laneId, worktree }) => goalId === target.goalId || laneId === target.laneId || worktree === target.worktree) ||
|
|
1159
1236
|
bindings.some(({ request }) => request.laneId === target.laneId ||
|
|
1160
1237
|
request.worker.agentId === target.worker.agentId ||
|
|
1161
|
-
request
|
|
1238
|
+
admissionEvidence(request)?.sessionId === target.sessionId ||
|
|
1162
1239
|
request.existingGoalRecovery?.sessionId === target.sessionId) ||
|
|
1163
1240
|
decisions.some(({ record }) => {
|
|
1164
1241
|
const handoff = Schema.decodeUnknownSync(WorkDecisionHandoff)(JSON.parse(record));
|
|
@@ -1186,6 +1263,185 @@ const readRecoveryState = (database, target) => {
|
|
|
1186
1263
|
throw cause;
|
|
1187
1264
|
}
|
|
1188
1265
|
};
|
|
1266
|
+
const ReassignmentApprovalRow = Schema.Struct({ approvalJobId: Schema.String, goalId: Schema.String });
|
|
1267
|
+
const ReconcilerEventRow = Schema.Struct({ eventId: Schema.String, goalId: Schema.String });
|
|
1268
|
+
const ObservedFactRow = Schema.Struct({
|
|
1269
|
+
subject: Schema.String,
|
|
1270
|
+
observationId: Schema.String,
|
|
1271
|
+
observedAt: Schema.Number,
|
|
1272
|
+
confirmedAt: Schema.Number,
|
|
1273
|
+
record: Schema.String
|
|
1274
|
+
});
|
|
1275
|
+
const StoredFactRow = Schema.Struct({ observationId: Schema.String, confirmedAt: Schema.Number });
|
|
1276
|
+
const FactTotalsRow = Schema.Struct({ count: Schema.Number, bytes: Schema.Number });
|
|
1277
|
+
const EvictionRow = Schema.Struct({ subject: Schema.String, bytes: Schema.Number, age: Schema.Number });
|
|
1278
|
+
const EvictedThroughRow = Schema.Struct({ evictedThrough: Schema.Number });
|
|
1279
|
+
const canonicalObservation = (observation) => observation._tag === "agent"
|
|
1280
|
+
? { ...observation, host: asciiLower(observation.host) }
|
|
1281
|
+
: { ...observation, repository: asciiLower(observation.repository) };
|
|
1282
|
+
/**
|
|
1283
|
+
* Applies each observation in one transaction, then evicts the oldest rows
|
|
1284
|
+
* until facts and failures are each within their bounds.
|
|
1285
|
+
*
|
|
1286
|
+
* - The same facts again keep their first-seen time and move `confirmed_at`.
|
|
1287
|
+
* - Different facts replace the stored ones, unless the stored ones were
|
|
1288
|
+
* confirmed at or after this observation (then it is stale).
|
|
1289
|
+
* - A good read ends the subject's run of failures. A failure starts a run,
|
|
1290
|
+
* or keeps the running one's `since`; one older than the last good read is stale.
|
|
1291
|
+
*/
|
|
1292
|
+
const writeObservations = (database, prepared) => {
|
|
1293
|
+
database.exec("BEGIN IMMEDIATE");
|
|
1294
|
+
try {
|
|
1295
|
+
const readFact = database.prepare(`SELECT observation_id AS observationId, confirmed_at AS confirmedAt
|
|
1296
|
+
FROM work_observed_facts WHERE subject = ?`);
|
|
1297
|
+
// The newest time of any row eviction has removed. A subject with no row
|
|
1298
|
+
// may have had one, so a fact read no newer than this can't be shown to
|
|
1299
|
+
// be current, and is stale: eviction never makes an old read new again.
|
|
1300
|
+
const evictedRow = database.prepare("SELECT evicted_through AS evictedThrough FROM work_observed_eviction WHERE singleton = 1").get();
|
|
1301
|
+
const evictedThrough = evictedRow === undefined
|
|
1302
|
+
? Number.NEGATIVE_INFINITY
|
|
1303
|
+
: Schema.decodeUnknownSync(EvictedThroughRow)(evictedRow).evictedThrough;
|
|
1304
|
+
const upsertFact = database.prepare(`INSERT INTO work_observed_facts (subject, observation_id, observed_at, confirmed_at, record)
|
|
1305
|
+
VALUES (?, ?, ?, ?, ?)
|
|
1306
|
+
ON CONFLICT (subject) DO UPDATE SET observation_id = excluded.observation_id,
|
|
1307
|
+
observed_at = excluded.observed_at, confirmed_at = excluded.confirmed_at, record = excluded.record`);
|
|
1308
|
+
const confirmFact = database.prepare("UPDATE work_observed_facts SET confirmed_at = max(confirmed_at, ?) WHERE subject = ?");
|
|
1309
|
+
// A good read ends a run of failures only if it is newer than the run's
|
|
1310
|
+
// latest failure.
|
|
1311
|
+
const clearFailures = database.prepare("DELETE FROM work_observed_failures WHERE subject = ? AND last_at <= ?");
|
|
1312
|
+
// We keep only the run's first and latest failure, so a run split by a
|
|
1313
|
+
// delayed good read restarts at its latest failure: a real failed read.
|
|
1314
|
+
const trimFailures = database.prepare("UPDATE work_observed_failures SET since = last_at WHERE subject = ? AND since <= ? AND last_at > ?");
|
|
1315
|
+
const endFailures = {
|
|
1316
|
+
run: (subject, observedAt) => {
|
|
1317
|
+
clearFailures.run(subject, observedAt);
|
|
1318
|
+
trimFailures.run(subject, observedAt, observedAt);
|
|
1319
|
+
}
|
|
1320
|
+
};
|
|
1321
|
+
// `since` is the earliest failure of the run; source and reason come from
|
|
1322
|
+
// the latest one, whatever order the failures arrive in.
|
|
1323
|
+
const recordFailure = database.prepare(`INSERT INTO work_observed_failures (subject, source, reason, since, last_at) VALUES (?, ?, ?, ?, ?)
|
|
1324
|
+
ON CONFLICT (subject) DO UPDATE SET
|
|
1325
|
+
since = min(since, excluded.since),
|
|
1326
|
+
source = CASE WHEN excluded.last_at >= last_at THEN excluded.source ELSE source END,
|
|
1327
|
+
reason = CASE WHEN excluded.last_at >= last_at THEN excluded.reason ELSE reason END,
|
|
1328
|
+
last_at = max(last_at, excluded.last_at)`);
|
|
1329
|
+
const outcomes = prepared.map((item) => {
|
|
1330
|
+
if (item._tag === "future")
|
|
1331
|
+
return { _tag: "stale", subject: item.subject };
|
|
1332
|
+
const row = readFact.get(item.subject);
|
|
1333
|
+
const stored = row === undefined ? undefined : Schema.decodeUnknownSync(StoredFactRow)(row);
|
|
1334
|
+
if (item._tag === "unknown") {
|
|
1335
|
+
if (stored !== undefined && stored.confirmedAt >= item.observedAt) {
|
|
1336
|
+
return { _tag: "stale", subject: item.subject };
|
|
1337
|
+
}
|
|
1338
|
+
recordFailure.run(item.subject, item.source, item.reason, item.observedAt, item.observedAt);
|
|
1339
|
+
return { _tag: "unknown", reason: item.reason, subject: item.subject };
|
|
1340
|
+
}
|
|
1341
|
+
if (stored === undefined && item.observedAt <= evictedThrough)
|
|
1342
|
+
return { _tag: "stale", subject: item.subject };
|
|
1343
|
+
// The same facts read before the stored confirmation confirm nothing now.
|
|
1344
|
+
if (stored?.observationId === item.observationId && stored.confirmedAt > item.observedAt) {
|
|
1345
|
+
return { _tag: "stale", subject: item.subject };
|
|
1346
|
+
}
|
|
1347
|
+
if (stored?.observationId === item.observationId) {
|
|
1348
|
+
confirmFact.run(item.observedAt, item.subject);
|
|
1349
|
+
endFailures.run(item.subject, item.observedAt);
|
|
1350
|
+
return { _tag: "unchanged", observationId: item.observationId, subject: item.subject };
|
|
1351
|
+
}
|
|
1352
|
+
if (stored !== undefined && stored.confirmedAt >= item.observedAt)
|
|
1353
|
+
return { _tag: "stale", subject: item.subject };
|
|
1354
|
+
upsertFact.run(item.subject, item.observationId, item.observedAt, item.observedAt, item.record);
|
|
1355
|
+
endFailures.run(item.subject, item.observedAt);
|
|
1356
|
+
return { _tag: "stored", observationId: item.observationId, subject: item.subject };
|
|
1357
|
+
});
|
|
1358
|
+
// Totals are read once and kept current as rows go, so eviction is one
|
|
1359
|
+
// indexed lookup and delete per row, not a full rescan per row.
|
|
1360
|
+
// A failure disputes its subject's fact while it is newer than that fact's
|
|
1361
|
+
// confirmation; evicting such a failure evicts the fact with it, so a
|
|
1362
|
+
// disputed fact never outlives the record of its dispute.
|
|
1363
|
+
const removeDisputedFact = database.prepare(`DELETE FROM work_observed_facts WHERE subject = ? AND confirmed_at <
|
|
1364
|
+
(SELECT last_at FROM work_observed_failures WHERE subject = ?)`);
|
|
1365
|
+
let newestEvicted = Number.NEGATIVE_INFINITY;
|
|
1366
|
+
const evict = (table, age, payload, beforeRemove) => {
|
|
1367
|
+
const size = `length(CAST(subject AS BLOB)) + length(CAST(${payload} AS BLOB))`;
|
|
1368
|
+
const totals = Schema.decodeUnknownSync(FactTotalsRow)(database.prepare(`SELECT COUNT(*) AS count, COALESCE(SUM(${size}), 0) AS bytes FROM ${table}`).get());
|
|
1369
|
+
const oldest = database.prepare(`SELECT subject, ${size} AS bytes, ${age} AS age FROM ${table} ORDER BY ${age} ASC, subject ASC LIMIT 1`);
|
|
1370
|
+
const remove = database.prepare(`DELETE FROM ${table} WHERE subject = ?`);
|
|
1371
|
+
let { bytes, count } = totals;
|
|
1372
|
+
let evicted = 0;
|
|
1373
|
+
while (count > workObservedFactMaxRecords || bytes > workObservedFactMaxBytes) {
|
|
1374
|
+
const row = Schema.decodeUnknownSync(EvictionRow)(oldest.get());
|
|
1375
|
+
newestEvicted = Math.max(newestEvicted, row.age);
|
|
1376
|
+
evicted += beforeRemove?.(row.subject) ?? 0;
|
|
1377
|
+
remove.run(row.subject);
|
|
1378
|
+
count -= 1;
|
|
1379
|
+
bytes -= row.bytes;
|
|
1380
|
+
evicted += 1;
|
|
1381
|
+
}
|
|
1382
|
+
return evicted;
|
|
1383
|
+
};
|
|
1384
|
+
// The least recently read rows go first: a fact by its last confirmation,
|
|
1385
|
+
// a failure by its latest failed read.
|
|
1386
|
+
const evicted = evict("work_observed_facts", "confirmed_at", "record") +
|
|
1387
|
+
evict("work_observed_failures", "last_at", "reason", (subject) => Number(removeDisputedFact.run(subject, subject).changes));
|
|
1388
|
+
if (newestEvicted > evictedThrough) {
|
|
1389
|
+
database.prepare(`INSERT INTO work_observed_eviction (singleton, evicted_through) VALUES (1, ?)
|
|
1390
|
+
ON CONFLICT (singleton) DO UPDATE SET evicted_through = max(evicted_through, excluded.evicted_through)`).run(newestEvicted);
|
|
1391
|
+
}
|
|
1392
|
+
// Reported last, after every write and eviction in this call: a stored or
|
|
1393
|
+
// unchanged fact that a later envelope replaced, or eviction removed, is no
|
|
1394
|
+
// longer the subject's fact and confirms nothing.
|
|
1395
|
+
// Only a subject's last accepted read can still be its fact: an earlier one
|
|
1396
|
+
// with the same content (A, B, A) was replaced in between.
|
|
1397
|
+
const lastAccepted = new Map();
|
|
1398
|
+
outcomes.forEach((outcome, index) => {
|
|
1399
|
+
if (outcome._tag === "stored" || outcome._tag === "unchanged")
|
|
1400
|
+
lastAccepted.set(outcome.subject, index);
|
|
1401
|
+
});
|
|
1402
|
+
const finalOutcomes = outcomes.map((outcome, index) => {
|
|
1403
|
+
if (outcome._tag !== "stored" && outcome._tag !== "unchanged")
|
|
1404
|
+
return outcome;
|
|
1405
|
+
if (lastAccepted.get(outcome.subject) !== index)
|
|
1406
|
+
return { _tag: "stale", subject: outcome.subject };
|
|
1407
|
+
const row = readFact.get(outcome.subject);
|
|
1408
|
+
const current = row === undefined ? undefined : Schema.decodeUnknownSync(StoredFactRow)(row);
|
|
1409
|
+
return current?.observationId === outcome.observationId ? outcome : { _tag: "stale", subject: outcome.subject };
|
|
1410
|
+
});
|
|
1411
|
+
database.exec("COMMIT");
|
|
1412
|
+
return { evicted, outcomes: finalOutcomes };
|
|
1413
|
+
}
|
|
1414
|
+
catch (cause) {
|
|
1415
|
+
if (database.isTransaction)
|
|
1416
|
+
database.exec("ROLLBACK");
|
|
1417
|
+
throw cause;
|
|
1418
|
+
}
|
|
1419
|
+
};
|
|
1420
|
+
const admissionSummary = (provenance) => provenance._tag === "approved"
|
|
1421
|
+
? `Prospective admission of an existing owner by approved Fleet job ${provenance.approvalJobId}`
|
|
1422
|
+
: `Admission of an observed worker by the reconciler from observation ${provenance.observationId}`;
|
|
1423
|
+
/** The binding-request key that records how a worker was admitted. */
|
|
1424
|
+
const admissionRecord = (provenance, evidence) => provenance._tag === "approved"
|
|
1425
|
+
? {
|
|
1426
|
+
prospectiveAdmission: {
|
|
1427
|
+
...evidence,
|
|
1428
|
+
approvalJobId: provenance.approvalJobId,
|
|
1429
|
+
approvalActor: provenance.approvalActor
|
|
1430
|
+
}
|
|
1431
|
+
}
|
|
1432
|
+
: { observedAdmission: { ...evidence, actor: "reconciler", observationId: provenance.observationId } };
|
|
1433
|
+
/** Whether a stored binding was admitted by exactly this provenance and absence evidence (an exact replay). */
|
|
1434
|
+
const sameAdmission = (request, expectedAbsenceToken, provenance) => {
|
|
1435
|
+
if (provenance._tag === "approved") {
|
|
1436
|
+
const stored = request.prospectiveAdmission;
|
|
1437
|
+
return stored !== undefined &&
|
|
1438
|
+
stored.expectedAbsenceToken === expectedAbsenceToken &&
|
|
1439
|
+
stored.approvalJobId === provenance.approvalJobId && stored.approvalActor === provenance.approvalActor;
|
|
1440
|
+
}
|
|
1441
|
+
const stored = request.observedAdmission;
|
|
1442
|
+
return stored !== undefined &&
|
|
1443
|
+
stored.expectedAbsenceToken === expectedAbsenceToken && stored.observationId === provenance.observationId;
|
|
1444
|
+
};
|
|
1189
1445
|
export class WorkStore {
|
|
1190
1446
|
#database;
|
|
1191
1447
|
#cryptoService;
|
|
@@ -1209,6 +1465,7 @@ export class WorkStore {
|
|
|
1209
1465
|
try {
|
|
1210
1466
|
migrateLegacyAuthorityTables(database);
|
|
1211
1467
|
const hadLaneOperationLedger = database.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'work_lane_operations'").get() !== undefined;
|
|
1468
|
+
const hadEvictionWatermark = database.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'work_observed_eviction'").get() !== undefined;
|
|
1212
1469
|
database.exec(`
|
|
1213
1470
|
CREATE TABLE IF NOT EXISTS work_goal_events (
|
|
1214
1471
|
event_id TEXT PRIMARY KEY,
|
|
@@ -1289,6 +1546,13 @@ export class WorkStore {
|
|
|
1289
1546
|
goal_id TEXT NOT NULL,
|
|
1290
1547
|
record TEXT NOT NULL
|
|
1291
1548
|
);
|
|
1549
|
+
-- One row per approved abandonment, committed with the goal checkpoint whose
|
|
1550
|
+
-- event id equals its approval job id, so work_goal_events bounds it too.
|
|
1551
|
+
CREATE TABLE IF NOT EXISTS work_goal_abandonments (
|
|
1552
|
+
approval_job_id TEXT PRIMARY KEY,
|
|
1553
|
+
goal_id TEXT NOT NULL,
|
|
1554
|
+
record TEXT NOT NULL
|
|
1555
|
+
);
|
|
1292
1556
|
CREATE TABLE IF NOT EXISTS work_decision_handoffs (
|
|
1293
1557
|
handoff_id TEXT PRIMARY KEY,
|
|
1294
1558
|
session_id TEXT NOT NULL UNIQUE,
|
|
@@ -1312,13 +1576,55 @@ export class WorkStore {
|
|
|
1312
1576
|
END;
|
|
1313
1577
|
CREATE INDEX IF NOT EXISTS work_decision_handoffs_lane_time
|
|
1314
1578
|
ON work_decision_handoffs (lane_id, occurred_at, handoff_id);
|
|
1315
|
-
CREATE INDEX IF NOT EXISTS work_decision_handoffs_session
|
|
1579
|
+
CREATE UNIQUE INDEX IF NOT EXISTS work_decision_handoffs_session
|
|
1316
1580
|
ON work_decision_handoffs (session_id);
|
|
1581
|
+
CREATE TABLE IF NOT EXISTS work_observed_facts (
|
|
1582
|
+
subject TEXT PRIMARY KEY,
|
|
1583
|
+
observation_id TEXT NOT NULL,
|
|
1584
|
+
observed_at INTEGER NOT NULL,
|
|
1585
|
+
confirmed_at INTEGER NOT NULL,
|
|
1586
|
+
record TEXT NOT NULL
|
|
1587
|
+
);
|
|
1588
|
+
CREATE INDEX IF NOT EXISTS work_observed_facts_age
|
|
1589
|
+
ON work_observed_facts (confirmed_at, subject);
|
|
1590
|
+
CREATE TABLE IF NOT EXISTS work_reconciler_events (
|
|
1591
|
+
event_id TEXT PRIMARY KEY,
|
|
1592
|
+
goal_id TEXT NOT NULL
|
|
1593
|
+
);
|
|
1594
|
+
CREATE TABLE IF NOT EXISTS work_observed_failures (
|
|
1595
|
+
subject TEXT PRIMARY KEY,
|
|
1596
|
+
source TEXT NOT NULL,
|
|
1597
|
+
reason TEXT NOT NULL,
|
|
1598
|
+
since INTEGER NOT NULL,
|
|
1599
|
+
last_at INTEGER NOT NULL
|
|
1600
|
+
);
|
|
1601
|
+
CREATE INDEX IF NOT EXISTS work_observed_failures_age
|
|
1602
|
+
ON work_observed_failures (last_at, subject);
|
|
1603
|
+
CREATE TABLE IF NOT EXISTS work_observed_eviction (
|
|
1604
|
+
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
|
|
1605
|
+
evicted_through INTEGER NOT NULL
|
|
1606
|
+
);
|
|
1317
1607
|
`);
|
|
1608
|
+
requireUniqueSessionIndex(database);
|
|
1318
1609
|
const columns = Schema.decodeUnknownSync(Schema.Array(Schema.Struct({ name: Schema.String })))(database.prepare("PRAGMA table_info(work_goal_events)").all());
|
|
1319
1610
|
if (!columns.some(({ name }) => name === "transaction_id")) {
|
|
1320
1611
|
database.exec("ALTER TABLE work_goal_events ADD COLUMN transaction_id TEXT");
|
|
1321
1612
|
}
|
|
1613
|
+
if (!hadEvictionWatermark) {
|
|
1614
|
+
// Older stores evicted without a watermark, and may have kept a fact
|
|
1615
|
+
// whose dispute they evicted. Their reads are a cache of the
|
|
1616
|
+
// providers, so they are dropped and every one becomes stale; the
|
|
1617
|
+
// next pass reads them again, newer.
|
|
1618
|
+
database.exec(`
|
|
1619
|
+
INSERT INTO work_observed_eviction (singleton, evicted_through)
|
|
1620
|
+
SELECT 1, max(at) FROM (
|
|
1621
|
+
SELECT confirmed_at AS at FROM work_observed_facts
|
|
1622
|
+
UNION ALL SELECT last_at FROM work_observed_failures
|
|
1623
|
+
) HAVING max(at) IS NOT NULL;
|
|
1624
|
+
DELETE FROM work_observed_facts;
|
|
1625
|
+
DELETE FROM work_observed_failures;
|
|
1626
|
+
`);
|
|
1627
|
+
}
|
|
1322
1628
|
if (!hadLaneOperationLedger) {
|
|
1323
1629
|
database.exec(`
|
|
1324
1630
|
INSERT OR IGNORE INTO work_lane_operations
|
|
@@ -1876,6 +2182,155 @@ export class WorkStore {
|
|
|
1876
2182
|
return yield* decision.error;
|
|
1877
2183
|
return decision.result;
|
|
1878
2184
|
});
|
|
2185
|
+
/**
|
|
2186
|
+
* Applies an approved `work.abandon` job: the goal, owned by exactly the
|
|
2187
|
+
* approved owner and still at the approved head, becomes `abandoned` with its
|
|
2188
|
+
* blocker cleared and a status activity naming the job. A goal with an
|
|
2189
|
+
* active lane, or already finished, is refused; nothing else is written. An
|
|
2190
|
+
* exact replay of the job returns the stored result.
|
|
2191
|
+
*/
|
|
2192
|
+
abandon = Effect.fn("WorkStore.abandon")(function* (request) {
|
|
2193
|
+
const decoded = yield* Schema.decodeUnknownEffect(WorkGoalAbandonment)(request).pipe(Effect.mapError(storeError("abandon.decode")));
|
|
2194
|
+
const jobId = decoded.approvalJobId;
|
|
2195
|
+
const observedAt = yield* Clock.currentTimeMillis;
|
|
2196
|
+
yield* this.secureFiles();
|
|
2197
|
+
const decision = yield* Effect.try({
|
|
2198
|
+
try: () => {
|
|
2199
|
+
let transaction = false;
|
|
2200
|
+
const reject = (error) => {
|
|
2201
|
+
this.#database.exec("ROLLBACK");
|
|
2202
|
+
transaction = false;
|
|
2203
|
+
return { _tag: "rejected", error };
|
|
2204
|
+
};
|
|
2205
|
+
try {
|
|
2206
|
+
this.#database.exec("BEGIN IMMEDIATE");
|
|
2207
|
+
transaction = true;
|
|
2208
|
+
const priorRaw = this.#database
|
|
2209
|
+
.prepare(`SELECT approval_job_id AS approvalJobId, goal_id AS goalId, record
|
|
2210
|
+
FROM work_goal_abandonments WHERE approval_job_id = ?`)
|
|
2211
|
+
.get(jobId);
|
|
2212
|
+
if (priorRaw !== undefined) {
|
|
2213
|
+
const row = Schema.decodeUnknownSync(ReassignmentRow)(priorRaw);
|
|
2214
|
+
const prior = Schema.decodeUnknownSync(WorkGoalAbandoned)(JSON.parse(row.record));
|
|
2215
|
+
if (row.approvalJobId !== prior.abandonment.approvalJobId || row.goalId !== prior.abandonment.goalId) {
|
|
2216
|
+
return reject(new WorkStoreError({ cause: { prior, row }, operation: "abandon.replay.identity-mismatch" }));
|
|
2217
|
+
}
|
|
2218
|
+
if (!Equal.equals(prior.abandonment, decoded)) {
|
|
2219
|
+
return reject(new WorkGoalAbandonmentConflictError({ approvalJobId: jobId, reason: "payload_mismatch" }));
|
|
2220
|
+
}
|
|
2221
|
+
this.#database.exec("ROLLBACK");
|
|
2222
|
+
transaction = false;
|
|
2223
|
+
return { _tag: "abandoned", result: prior };
|
|
2224
|
+
}
|
|
2225
|
+
const collision = this.#database
|
|
2226
|
+
.prepare(`SELECT 1 FROM work_goal_events WHERE event_id = ?
|
|
2227
|
+
UNION ALL SELECT 1 FROM work_lane_operations WHERE operation_id = ?
|
|
2228
|
+
UNION ALL SELECT 1 FROM work_agent_bindings WHERE dispatch_request_id = ? LIMIT 1`)
|
|
2229
|
+
.get(jobId, jobId, jobId);
|
|
2230
|
+
if (collision !== undefined) {
|
|
2231
|
+
return reject(new WorkGoalAbandonmentConflictError({ approvalJobId: jobId, reason: "identifier_in_use" }));
|
|
2232
|
+
}
|
|
2233
|
+
const eventRows = Schema.decodeUnknownSync(Schema.Array(AgentBindingGoalEventRow))(this.#database
|
|
2234
|
+
.prepare(`SELECT event_id AS eventId, goal_id AS goalId, occurred_at AS occurredAt, record
|
|
2235
|
+
FROM work_goal_events ORDER BY occurred_at ASC, event_id ASC LIMIT ?`)
|
|
2236
|
+
.all(workHistoryMaxEvents + 1));
|
|
2237
|
+
const history = [];
|
|
2238
|
+
for (const row of eventRows) {
|
|
2239
|
+
const event = decodeAgentBindingGoalEvent(row, "abandon.history");
|
|
2240
|
+
if (event._tag === "invalid")
|
|
2241
|
+
return reject(event.error);
|
|
2242
|
+
history.push(event.checkpoint);
|
|
2243
|
+
}
|
|
2244
|
+
const head = history.filter(({ goal }) => goal.id === decoded.goalId).at(-1);
|
|
2245
|
+
if (head === undefined ||
|
|
2246
|
+
head.eventId !== decoded.expectedGoalEventId ||
|
|
2247
|
+
head.goal.updatedAt !== decoded.expectedGoalUpdatedAt) {
|
|
2248
|
+
return reject(new WorkGoalRevisionConflictError({
|
|
2249
|
+
goalId: decoded.goalId,
|
|
2250
|
+
expectedEventId: decoded.expectedGoalEventId,
|
|
2251
|
+
expectedUpdatedAt: decoded.expectedGoalUpdatedAt,
|
|
2252
|
+
actualEventId: head?.eventId ?? null,
|
|
2253
|
+
actualUpdatedAt: head?.goal.updatedAt ?? null
|
|
2254
|
+
}));
|
|
2255
|
+
}
|
|
2256
|
+
if (!Equal.equals(head.goal.owner, decoded.owner)) {
|
|
2257
|
+
return reject(new WorkGoalOwnerMismatchError({
|
|
2258
|
+
goalId: decoded.goalId,
|
|
2259
|
+
laneId: null,
|
|
2260
|
+
expectedOwner: { id: decoded.owner.id, name: decoded.owner.name },
|
|
2261
|
+
actualOwner: { id: head.goal.owner.id, name: head.goal.owner.name }
|
|
2262
|
+
}));
|
|
2263
|
+
}
|
|
2264
|
+
if (isTerminalWorkState(head.goal.state)) {
|
|
2265
|
+
return reject(new WorkGoalTerminalError({ goalId: decoded.goalId, state: head.goal.state }));
|
|
2266
|
+
}
|
|
2267
|
+
const ledger = readValidatedLaneLedger(this.#database, "abandon.lane");
|
|
2268
|
+
if (ledger._tag === "invalid")
|
|
2269
|
+
return reject(ledger.error);
|
|
2270
|
+
const active = ledger.entries.find(({ claim }) => claim.goalId === decoded.goalId && claim.phase !== "shipped");
|
|
2271
|
+
if (active !== undefined) {
|
|
2272
|
+
return reject(new WorkGoalLaneActiveError({ goalId: decoded.goalId, laneId: active.claim.laneId }));
|
|
2273
|
+
}
|
|
2274
|
+
// occurredAt equals updatedAt for every valid checkpoint; both bound the ordering defensively.
|
|
2275
|
+
const at = Math.max(observedAt, head.occurredAt + 1, head.goal.updatedAt + 1);
|
|
2276
|
+
const existingActivity = head.goal.activity ?? [];
|
|
2277
|
+
const goal = Schema.decodeUnknownSync(WorkGoal)({
|
|
2278
|
+
...withoutBlockers(head.goal),
|
|
2279
|
+
state: "abandoned",
|
|
2280
|
+
// A full activity list keeps the owner's entries and takes no note; the
|
|
2281
|
+
// checkpoint's event id and the abandonments table still name the job.
|
|
2282
|
+
activity: existingActivity.length >= goalActivityLimit ? existingActivity : [
|
|
2283
|
+
...existingActivity,
|
|
2284
|
+
{
|
|
2285
|
+
id: jobId,
|
|
2286
|
+
kind: "status",
|
|
2287
|
+
summary: workAbandonActivitySummary(decoded, jobId, decoded.approvalHash),
|
|
2288
|
+
occurredAt: at
|
|
2289
|
+
}
|
|
2290
|
+
],
|
|
2291
|
+
updatedAt: at
|
|
2292
|
+
});
|
|
2293
|
+
const checkpoint = Schema.decodeUnknownSync(WorkGoalCheckpoint)({
|
|
2294
|
+
version: "herdr.work.event.v1",
|
|
2295
|
+
eventId: jobId,
|
|
2296
|
+
occurredAt: at,
|
|
2297
|
+
goal
|
|
2298
|
+
});
|
|
2299
|
+
const result = Schema.decodeUnknownSync(WorkGoalAbandoned)({ abandonment: decoded, checkpoint });
|
|
2300
|
+
const familyError = validateGoalFamilyHistory([...history, checkpoint]);
|
|
2301
|
+
if (familyError !== undefined)
|
|
2302
|
+
return reject(familyError);
|
|
2303
|
+
const capacity = agentBindingAdmissionError({
|
|
2304
|
+
history,
|
|
2305
|
+
candidate: checkpoint,
|
|
2306
|
+
operationCount: 0,
|
|
2307
|
+
operationBytes: 0,
|
|
2308
|
+
candidateOperationBytes: 0
|
|
2309
|
+
});
|
|
2310
|
+
if (capacity !== undefined)
|
|
2311
|
+
return reject(capacity);
|
|
2312
|
+
this.#database
|
|
2313
|
+
.prepare("INSERT INTO work_goal_events (event_id, goal_id, occurred_at, record) VALUES (?, ?, ?, ?)")
|
|
2314
|
+
.run(checkpoint.eventId, checkpoint.goal.id, checkpoint.occurredAt, JSON.stringify(checkpoint));
|
|
2315
|
+
this.#database
|
|
2316
|
+
.prepare("INSERT INTO work_goal_abandonments (approval_job_id, goal_id, record) VALUES (?, ?, ?)")
|
|
2317
|
+
.run(jobId, decoded.goalId, JSON.stringify(result));
|
|
2318
|
+
this.#database.exec("COMMIT");
|
|
2319
|
+
transaction = false;
|
|
2320
|
+
return { _tag: "abandoned", result };
|
|
2321
|
+
}
|
|
2322
|
+
catch (cause) {
|
|
2323
|
+
if (transaction)
|
|
2324
|
+
this.#database.exec("ROLLBACK");
|
|
2325
|
+
throw cause;
|
|
2326
|
+
}
|
|
2327
|
+
},
|
|
2328
|
+
catch: storeError("abandon.transaction")
|
|
2329
|
+
});
|
|
2330
|
+
if (decision._tag === "rejected")
|
|
2331
|
+
return yield* decision.error;
|
|
2332
|
+
return decision.result;
|
|
2333
|
+
});
|
|
1879
2334
|
admissionPreflight = Effect.fn("WorkStore.admissionPreflight")(function* (target) {
|
|
1880
2335
|
const decoded = yield* Schema.decodeUnknownEffect(WorkAdmissionTarget)(target).pipe(Effect.mapError(storeError("admission.preflight.decode")));
|
|
1881
2336
|
const state = yield* Effect.try({
|
|
@@ -1888,8 +2343,27 @@ export class WorkStore {
|
|
|
1888
2343
|
const preflight = { _tag: "prospective", target: state.target, absenceToken };
|
|
1889
2344
|
return preflight;
|
|
1890
2345
|
});
|
|
2346
|
+
/**
|
|
2347
|
+
* Admits an existing owner after an approved Fleet job. Its absence token is
|
|
2348
|
+
* re-checked inside the write; an exact replay returns the stored link.
|
|
2349
|
+
*/
|
|
1891
2350
|
admitExistingOwner = Effect.fn("WorkStore.admitExistingOwner")(function* (request) {
|
|
1892
2351
|
const decoded = yield* Schema.decodeUnknownEffect(WorkProspectiveAdmission)(request).pipe(Effect.mapError(storeError("admission.decode")));
|
|
2352
|
+
return yield* this.#admit(decoded, {
|
|
2353
|
+
_tag: "approved",
|
|
2354
|
+
approvalActor: decoded.approvalActor,
|
|
2355
|
+
approvalJobId: decoded.approvalJobId
|
|
2356
|
+
});
|
|
2357
|
+
});
|
|
2358
|
+
/**
|
|
2359
|
+
* Admits a worker the reconciler observed, without an approval: the same
|
|
2360
|
+
* write and checks as `admitExistingOwner`, credited to the observation.
|
|
2361
|
+
*/
|
|
2362
|
+
admitObserved = Effect.fn("WorkStore.admitObserved")(function* (request) {
|
|
2363
|
+
const decoded = yield* Schema.decodeUnknownEffect(WorkObservedAdmission)(request).pipe(Effect.mapError(storeError("admission.decode")));
|
|
2364
|
+
return yield* this.#admit(decoded, { _tag: "observed", observationId: decoded.observationId });
|
|
2365
|
+
});
|
|
2366
|
+
#admit = Effect.fn("WorkStore.admit")(function* (decoded, provenance) {
|
|
1893
2367
|
const target = Schema.decodeUnknownSync(WorkAdmissionTarget)(decoded);
|
|
1894
2368
|
const now = yield* Clock.currentTimeMillis;
|
|
1895
2369
|
yield* this.secureFiles();
|
|
@@ -1913,13 +2387,10 @@ export class WorkStore {
|
|
|
1913
2387
|
const state = admissionState(this.#database, target);
|
|
1914
2388
|
if (state._tag === "existing") {
|
|
1915
2389
|
const existing = state.link;
|
|
1916
|
-
const provenance = existing.binding.request.prospectiveAdmission;
|
|
1917
2390
|
this.#database.exec("ROLLBACK");
|
|
1918
2391
|
transaction = false;
|
|
1919
2392
|
return existing.binding.request.dispatchRequestId === decoded.operationId &&
|
|
1920
|
-
|
|
1921
|
-
provenance?.approvalJobId === decoded.approvalJobId &&
|
|
1922
|
-
provenance.approvalActor === decoded.approvalActor &&
|
|
2393
|
+
sameAdmission(existing.binding.request, decoded.expectedAbsenceToken, provenance) &&
|
|
1923
2394
|
existing.goal.title === decoded.title &&
|
|
1924
2395
|
existing.goal.summary === decoded.summary &&
|
|
1925
2396
|
existing.goal.detail === decoded.detail
|
|
@@ -1962,7 +2433,7 @@ export class WorkStore {
|
|
|
1962
2433
|
activity: [{
|
|
1963
2434
|
id: `${decoded.operationId}.admission`,
|
|
1964
2435
|
kind: "status",
|
|
1965
|
-
summary:
|
|
2436
|
+
summary: admissionSummary(provenance),
|
|
1966
2437
|
occurredAt: createdAt
|
|
1967
2438
|
}],
|
|
1968
2439
|
review: { state: "requested", summary: null, updatedAt: createdAt, url: decoded.reviewUrl },
|
|
@@ -1996,14 +2467,12 @@ export class WorkStore {
|
|
|
1996
2467
|
laneId: decoded.laneId,
|
|
1997
2468
|
expectedRevision: 0,
|
|
1998
2469
|
worker: decoded.worker,
|
|
1999
|
-
|
|
2470
|
+
...admissionRecord(provenance, {
|
|
2000
2471
|
sessionId: decoded.sessionId,
|
|
2001
2472
|
workAssignment: decoded.expectedWork,
|
|
2002
2473
|
baseHead: decoded.baseHead,
|
|
2003
|
-
expectedAbsenceToken: decoded.expectedAbsenceToken
|
|
2004
|
-
|
|
2005
|
-
approvalActor: decoded.approvalActor
|
|
2006
|
-
}
|
|
2474
|
+
expectedAbsenceToken: decoded.expectedAbsenceToken
|
|
2475
|
+
})
|
|
2007
2476
|
},
|
|
2008
2477
|
lane,
|
|
2009
2478
|
checkpoint: {
|
|
@@ -2050,7 +2519,7 @@ export class WorkStore {
|
|
|
2050
2519
|
_tag: "rejected",
|
|
2051
2520
|
error: capacity ?? new WorkProjectionError({
|
|
2052
2521
|
cause: decoded,
|
|
2053
|
-
detail: "
|
|
2522
|
+
detail: "admission exceeds Work capacity",
|
|
2054
2523
|
reason: "capacity_exceeded"
|
|
2055
2524
|
})
|
|
2056
2525
|
};
|
|
@@ -2197,7 +2666,7 @@ export class WorkStore {
|
|
|
2197
2666
|
if (currentDecision._tag === "invalid")
|
|
2198
2667
|
return reject(currentDecision.error);
|
|
2199
2668
|
const current = currentDecision.checkpoint;
|
|
2200
|
-
if (current.goal.state === "completed") {
|
|
2669
|
+
if (current.goal.state === "completed" || current.goal.state === "abandoned") {
|
|
2201
2670
|
return reject(new WorkAgentBindingAuthorityError({
|
|
2202
2671
|
actualRevision: lane.revision,
|
|
2203
2672
|
expectedRevision: decoded.expectedRevision,
|
|
@@ -2334,8 +2803,26 @@ export class WorkStore {
|
|
|
2334
2803
|
const entries = yield* Schema.decodeUnknownEffect(Schema.Array(Schema.Struct({ dispatchRequestId: WorkAgentBindingRequest.fields.dispatchRequestId })))(rows).pipe(Effect.mapError(storeError("agent-binding.lane.rows")));
|
|
2335
2804
|
return entries.map(({ dispatchRequestId }) => dispatchRequestId);
|
|
2336
2805
|
});
|
|
2337
|
-
append =
|
|
2806
|
+
append = (event) => this.appendAt(event, null).pipe(
|
|
2807
|
+
// Only an expected head can raise it; a plain append never checks one.
|
|
2808
|
+
Effect.catchTag("WorkGoalRevisionConflictError", Effect.die));
|
|
2809
|
+
/**
|
|
2810
|
+
* Appends one checkpoint. With `expectedHead`, the goal's latest checkpoint
|
|
2811
|
+
* must still be that one when the write commits (an exact replay always
|
|
2812
|
+
* succeeds), so a writer that planned from the head cannot bury a newer
|
|
2813
|
+
* owner checkpoint.
|
|
2814
|
+
*/
|
|
2815
|
+
appendAt = Effect.fn("WorkStore.append")(function* (event, guard) {
|
|
2338
2816
|
const decoded = yield* Schema.decodeUnknownEffect(WorkGoalCheckpoint)(event).pipe(Effect.mapError(storeError("append.decode")));
|
|
2817
|
+
// The reconciler's event and activity ids are its provenance; no other
|
|
2818
|
+
// writer may use them.
|
|
2819
|
+
if (guard === null && decoded.eventId.startsWith(reconcilerEventPrefix)) {
|
|
2820
|
+
return yield* new WorkProjectionError({
|
|
2821
|
+
cause: decoded.eventId,
|
|
2822
|
+
detail: `event ids starting with ${reconcilerEventPrefix} are reserved for the reconciler`,
|
|
2823
|
+
reason: "malformed"
|
|
2824
|
+
});
|
|
2825
|
+
}
|
|
2339
2826
|
yield* this.secureFiles();
|
|
2340
2827
|
const decision = yield* Effect.try({
|
|
2341
2828
|
try: () => {
|
|
@@ -2352,12 +2839,44 @@ export class WorkStore {
|
|
|
2352
2839
|
.prepare(`SELECT record FROM work_goal_events
|
|
2353
2840
|
WHERE event_id = ? OR (goal_id = ? AND occurred_at = ?)`)
|
|
2354
2841
|
.all(decoded.eventId, decoded.goal.id, decoded.occurredAt)).map(({ record }) => Schema.decodeUnknownSync(WorkGoalCheckpoint)(JSON.parse(record)));
|
|
2355
|
-
if (collisions.length > 0) {
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2842
|
+
if (collisions.length > 0 && collisions.every((existing) => Equal.equals(existing, decoded))) {
|
|
2843
|
+
this.#database.exec("ROLLBACK");
|
|
2844
|
+
transaction = false;
|
|
2845
|
+
return { _tag: "replayed", event: decoded };
|
|
2846
|
+
}
|
|
2847
|
+
if (guard !== null) {
|
|
2848
|
+
// Planned from this head and this fact: if either moved, the plan is stale.
|
|
2849
|
+
const headRow = this.#database.prepare(`SELECT record FROM work_goal_events WHERE goal_id = ?
|
|
2850
|
+
ORDER BY occurred_at DESC, event_id DESC LIMIT 1`).get(decoded.goal.id);
|
|
2851
|
+
const head = headRow === undefined ? null : Schema.decodeUnknownSync(WorkGoalCheckpoint)(JSON.parse(Schema.decodeUnknownSync(StoredEventRow)(headRow).record));
|
|
2852
|
+
const factRow = this.#database.prepare(`SELECT observation_id AS observationId, confirmed_at AS confirmedAt
|
|
2853
|
+
FROM work_observed_facts WHERE subject = ?`).get(guard.fact.subject);
|
|
2854
|
+
const fact = factRow === undefined ? null : Schema.decodeUnknownSync(ObservationIdRow)(factRow);
|
|
2855
|
+
const failureRow = this.#database.prepare("SELECT last_at AS lastAt FROM work_observed_failures WHERE subject = ?").get(guard.fact.subject);
|
|
2856
|
+
const failedAt = failureRow === undefined
|
|
2857
|
+
? null
|
|
2858
|
+
: Schema.decodeUnknownSync(FailureLastAtRow)(failureRow).lastAt;
|
|
2859
|
+
// A failed read recorded while the plan was made also makes it stale.
|
|
2860
|
+
const stillFresh = fact !== null && (failedAt === null || failedAt <= fact.confirmedAt);
|
|
2861
|
+
if (head?.eventId !== guard.head.eventId || fact?.observationId !== guard.fact.observationId || !stillFresh) {
|
|
2862
|
+
return reject(new WorkGoalRevisionConflictError({
|
|
2863
|
+
actualEventId: head?.eventId ?? null,
|
|
2864
|
+
actualUpdatedAt: head?.goal.updatedAt ?? null,
|
|
2865
|
+
expectedEventId: guard.head.eventId,
|
|
2866
|
+
expectedUpdatedAt: guard.head.goal.updatedAt,
|
|
2867
|
+
goalId: decoded.goal.id
|
|
2868
|
+
}));
|
|
2360
2869
|
}
|
|
2870
|
+
const count = Schema.decodeUnknownSync(CountRow)(this.#database.prepare("SELECT COUNT(*) AS count FROM work_goal_events").get()).count;
|
|
2871
|
+
if (count + 1 > workHistoryMaxEvents - guard.reserve) {
|
|
2872
|
+
return reject(new WorkProjectionError({
|
|
2873
|
+
cause: count,
|
|
2874
|
+
detail: `the reconciler leaves ${guard.reserve} of ${workHistoryMaxEvents} checkpoints free`,
|
|
2875
|
+
reason: "capacity_exceeded"
|
|
2876
|
+
}));
|
|
2877
|
+
}
|
|
2878
|
+
}
|
|
2879
|
+
if (collisions.length > 0) {
|
|
2361
2880
|
return reject(new WorkCheckpointConflictError({
|
|
2362
2881
|
eventId: decoded.eventId,
|
|
2363
2882
|
goalId: decoded.goal.id,
|
|
@@ -2412,6 +2931,11 @@ export class WorkStore {
|
|
|
2412
2931
|
}
|
|
2413
2932
|
}
|
|
2414
2933
|
const result = this.#database.prepare("INSERT INTO work_goal_events (event_id, goal_id, occurred_at, record) VALUES (?, ?, ?, ?)").run(decoded.eventId, decoded.goal.id, decoded.occurredAt, JSON.stringify(decoded));
|
|
2934
|
+
// The reconciler's own record of what it wrote: its authorship and its
|
|
2935
|
+
// once-per-goal rule come from this table, never from an id's spelling.
|
|
2936
|
+
if (guard !== null) {
|
|
2937
|
+
this.#database.prepare("INSERT INTO work_reconciler_events (event_id, goal_id) VALUES (?, ?)").run(decoded.eventId, decoded.goal.id);
|
|
2938
|
+
}
|
|
2415
2939
|
this.#database.exec("COMMIT");
|
|
2416
2940
|
transaction = false;
|
|
2417
2941
|
return { _tag: "inserted", changes: result.changes };
|
|
@@ -2443,6 +2967,14 @@ export class WorkStore {
|
|
|
2443
2967
|
}
|
|
2444
2968
|
const transaction = yield* Schema.decodeUnknownEffect(TransactionId)(transactionId).pipe(Effect.mapError(storeError("appendMany.decode.transaction")));
|
|
2445
2969
|
const decoded = yield* Effect.forEach(events, (event) => Schema.decodeUnknownEffect(WorkGoalCheckpoint)(event).pipe(Effect.mapError(storeError("appendMany.decode.event"))));
|
|
2970
|
+
const reserved = decoded.find(({ eventId }) => eventId.startsWith(reconcilerEventPrefix));
|
|
2971
|
+
if (reserved !== undefined) {
|
|
2972
|
+
return yield* new WorkProjectionError({
|
|
2973
|
+
cause: reserved.eventId,
|
|
2974
|
+
detail: `event ids starting with ${reconcilerEventPrefix} are reserved for the reconciler`,
|
|
2975
|
+
reason: "malformed"
|
|
2976
|
+
});
|
|
2977
|
+
}
|
|
2446
2978
|
if (decoded.length === 0) {
|
|
2447
2979
|
return yield* new WorkProjectionError({
|
|
2448
2980
|
cause: events,
|
|
@@ -2944,7 +3476,7 @@ export class WorkStore {
|
|
|
2944
3476
|
goal.goalFamily?.role !== "canonical" ||
|
|
2945
3477
|
goal.goalFamily.canonicalGoalId !== goal.id)
|
|
2946
3478
|
return rejectLink("missing_provenance");
|
|
2947
|
-
if (goal.state
|
|
3479
|
+
if (isTerminalWorkState(goal.state))
|
|
2948
3480
|
return rejectLink("terminal_goal");
|
|
2949
3481
|
if (!Equal.equals(goal.owner, reconciliation.expectedOwner))
|
|
2950
3482
|
return rejectLink("owner_mismatch");
|
|
@@ -3360,9 +3892,18 @@ export class WorkStore {
|
|
|
3360
3892
|
phase, revision, record
|
|
3361
3893
|
FROM work_lane_operations ORDER BY operation_id ASC
|
|
3362
3894
|
LIMIT ?`).all(workLaneOperationMaxRecords + 1);
|
|
3895
|
+
const facts = this.#database.prepare(`SELECT subject, observation_id AS observationId, observed_at AS observedAt,
|
|
3896
|
+
confirmed_at AS confirmedAt, record
|
|
3897
|
+
FROM work_observed_facts ORDER BY subject ASC
|
|
3898
|
+
LIMIT ?`).all(workObservedFactMaxRecords + 1);
|
|
3899
|
+
const failures = this.#database.prepare(`SELECT subject, source, reason, since, last_at AS lastAt
|
|
3900
|
+
FROM work_observed_failures ORDER BY subject ASC LIMIT ?`).all(workObservedFactMaxRecords + 1);
|
|
3901
|
+
const reconcilerRows = this.#database.prepare("SELECT event_id AS eventId, goal_id AS goalId FROM work_reconciler_events ORDER BY event_id ASC LIMIT ?").all(workHistoryMaxEvents + 1);
|
|
3902
|
+
const reassignments = this.#database.prepare(`SELECT approval_job_id AS approvalJobId, goal_id AS goalId FROM work_goal_reassignments
|
|
3903
|
+
UNION ALL SELECT approval_job_id AS approvalJobId, goal_id AS goalId FROM work_goal_abandonments`).all();
|
|
3363
3904
|
this.#database.exec("COMMIT");
|
|
3364
3905
|
inTransaction = false;
|
|
3365
|
-
return { bindings, events, laneOperations };
|
|
3906
|
+
return { bindings, events, facts, failures, laneOperations, reassignments, reconcilerRows };
|
|
3366
3907
|
}
|
|
3367
3908
|
catch (cause) {
|
|
3368
3909
|
if (inTransaction)
|
|
@@ -3387,6 +3928,23 @@ export class WorkStore {
|
|
|
3387
3928
|
const eventById = new Map(eventRows.map((row) => [row.eventId, row]));
|
|
3388
3929
|
const laneByOperation = new Map(laneRows.map((row) => [row.operationId, row]));
|
|
3389
3930
|
let logicalObservedAt = null;
|
|
3931
|
+
// An approved job's activity id is its operation (recovery), the operation
|
|
3932
|
+
// plus `.admission` (prospective admission), or the job id (reassignment).
|
|
3933
|
+
const reassignmentRows = yield* Schema.decodeUnknownEffect(Schema.Array(ReassignmentApprovalRow))(source.reassignments).pipe(Effect.mapError(storeError("snapshot-input.decode-reassignments")));
|
|
3934
|
+
// An approved reassignment or abandonment may be stamped past the clock
|
|
3935
|
+
// (one past a future-dated head); like a binding, it moves logical time so
|
|
3936
|
+
// a default snapshot shows what the approved job wrote.
|
|
3937
|
+
for (const { approvalJobId } of reassignmentRows) {
|
|
3938
|
+
const written = eventById.get(approvalJobId);
|
|
3939
|
+
if (written !== undefined)
|
|
3940
|
+
logicalObservedAt = Math.max(logicalObservedAt ?? 0, written.occurredAt);
|
|
3941
|
+
}
|
|
3942
|
+
const approvals = reassignmentRows.map(({ approvalJobId, goalId }) => ({
|
|
3943
|
+
activityId: approvalJobId,
|
|
3944
|
+
approvalJobId,
|
|
3945
|
+
goalId
|
|
3946
|
+
}));
|
|
3947
|
+
const observedAdmissions = [];
|
|
3390
3948
|
for (const row of bindingRows) {
|
|
3391
3949
|
const binding = yield* Effect.try({
|
|
3392
3950
|
try: () => Schema.decodeUnknownSync(WorkAgentBinding)(JSON.parse(row.record)),
|
|
@@ -3407,8 +3965,181 @@ export class WorkStore {
|
|
|
3407
3965
|
if (readbackError !== undefined)
|
|
3408
3966
|
return yield* readbackError;
|
|
3409
3967
|
logicalObservedAt = Math.max(logicalObservedAt ?? 0, binding.checkpoint.occurredAt);
|
|
3968
|
+
const goalId = binding.checkpoint.goal.id;
|
|
3969
|
+
const { dispatchRequestId, existingGoalRecovery, observedAdmission, ownerReassignment, prospectiveAdmission } = binding.request;
|
|
3970
|
+
if (observedAdmission !== undefined) {
|
|
3971
|
+
observedAdmissions.push({ eventId: `${dispatchRequestId}.admission`, goalId });
|
|
3972
|
+
}
|
|
3973
|
+
if (prospectiveAdmission !== undefined) {
|
|
3974
|
+
approvals.push({
|
|
3975
|
+
activityId: `${dispatchRequestId}.admission`,
|
|
3976
|
+
approvalJobId: prospectiveAdmission.approvalJobId,
|
|
3977
|
+
goalId
|
|
3978
|
+
});
|
|
3979
|
+
}
|
|
3980
|
+
if (existingGoalRecovery !== undefined) {
|
|
3981
|
+
approvals.push({ activityId: dispatchRequestId, approvalJobId: existingGoalRecovery.approvalJobId, goalId });
|
|
3982
|
+
}
|
|
3983
|
+
if (ownerReassignment !== undefined) {
|
|
3984
|
+
approvals.push({
|
|
3985
|
+
activityId: ownerReassignment.approvalJobId,
|
|
3986
|
+
approvalJobId: ownerReassignment.approvalJobId,
|
|
3987
|
+
goalId
|
|
3988
|
+
});
|
|
3989
|
+
}
|
|
3990
|
+
}
|
|
3991
|
+
const factRows = yield* Schema.decodeUnknownEffect(Schema.Array(ObservedFactRow))(source.facts).pipe(Effect.mapError(storeError("snapshot-input.decode-fact-rows")));
|
|
3992
|
+
const failures = yield* Schema.decodeUnknownEffect(Schema.Array(WorkObservedFailure))(source.failures).pipe(Effect.mapError(storeError("snapshot-input.decode-failures")));
|
|
3993
|
+
if (factRows.length > workObservedFactMaxRecords || failures.length > workObservedFactMaxRecords) {
|
|
3994
|
+
return yield* new WorkStoreError({ cause: factRows.length, operation: "snapshot-input.fact-capacity" });
|
|
3995
|
+
}
|
|
3996
|
+
const facts = yield* Effect.forEach(factRows, (row) => Effect.try({
|
|
3997
|
+
try: () => Schema.decodeUnknownSync(WorkObservedFact)({
|
|
3998
|
+
confirmedAt: row.confirmedAt,
|
|
3999
|
+
observation: JSON.parse(row.record),
|
|
4000
|
+
observationId: row.observationId,
|
|
4001
|
+
observedAt: row.observedAt,
|
|
4002
|
+
subject: row.subject
|
|
4003
|
+
}),
|
|
4004
|
+
catch: storeError("snapshot-input.decode-fact")
|
|
4005
|
+
}));
|
|
4006
|
+
// A row is filed under its own facts' subject; anything else is a corrupt row.
|
|
4007
|
+
const misfiled = facts.find((fact) => fact.subject !== observationSubject(fact.observation));
|
|
4008
|
+
if (misfiled !== undefined) {
|
|
4009
|
+
return yield* new WorkStoreError({ cause: misfiled.subject, operation: "snapshot-input.fact-subject" });
|
|
4010
|
+
}
|
|
4011
|
+
const reconcilerEvents = yield* Schema.decodeUnknownEffect(Schema.Array(ReconcilerEventRow))(source.reconcilerRows)
|
|
4012
|
+
.pipe(Effect.mapError(storeError("snapshot-input.decode-reconciler-events")));
|
|
4013
|
+
// Each current activity's origin: the checkpoint that started its current
|
|
4014
|
+
// unchanged run. An activity that disappears, or reads differently, starts
|
|
4015
|
+
// a new run, so a later owner rewrite or re-creation is never credited to
|
|
4016
|
+
// whoever wrote an earlier version.
|
|
4017
|
+
const activityOrigins = new Map();
|
|
4018
|
+
const previousByGoal = new Map();
|
|
4019
|
+
const ordered = events.toSorted((left, right) => left.occurredAt - right.occurredAt || (left.eventId < right.eventId ? -1 : left.eventId > right.eventId ? 1 : 0));
|
|
4020
|
+
for (const event of ordered) {
|
|
4021
|
+
const previous = previousByGoal.get(event.goal.id) ?? new Map();
|
|
4022
|
+
const current = new Map((event.goal.activity ?? []).map((activity) => [activity.id, activity]));
|
|
4023
|
+
for (const [id, activity] of current) {
|
|
4024
|
+
const before = previous.get(id);
|
|
4025
|
+
const unchanged = before !== undefined && before.kind === activity.kind &&
|
|
4026
|
+
before.summary === activity.summary &&
|
|
4027
|
+
before.occurredAt === activity.occurredAt;
|
|
4028
|
+
if (!unchanged)
|
|
4029
|
+
activityOrigins.set(`${event.goal.id}\u0000${id}`, { activity, eventId: event.eventId });
|
|
4030
|
+
}
|
|
4031
|
+
previousByGoal.set(event.goal.id, current);
|
|
4032
|
+
}
|
|
4033
|
+
return {
|
|
4034
|
+
activityOrigins,
|
|
4035
|
+
approvals,
|
|
4036
|
+
events,
|
|
4037
|
+
facts,
|
|
4038
|
+
failures,
|
|
4039
|
+
logicalObservedAt,
|
|
4040
|
+
observedAdmissions,
|
|
4041
|
+
reconcilerEvents
|
|
4042
|
+
};
|
|
4043
|
+
});
|
|
4044
|
+
reconcile = Effect.fn("WorkStore.reconcile")(function* (request) {
|
|
4045
|
+
const options = yield* Schema.decodeUnknownEffect(WorkReconcileOptions)(request).pipe(Effect.mapError(storeError("reconcile.options")));
|
|
4046
|
+
const source = yield* this.snapshotInput();
|
|
4047
|
+
if (workHistoryMaxEvents - source.events.length < workReconcilerHeadroom) {
|
|
4048
|
+
return yield* new WorkProjectionError({
|
|
4049
|
+
cause: source.events.length,
|
|
4050
|
+
detail: `the reconciler leaves ${workReconcilerHeadroom} of ${workHistoryMaxEvents} checkpoints free`,
|
|
4051
|
+
reason: "capacity_exceeded"
|
|
4052
|
+
});
|
|
3410
4053
|
}
|
|
3411
|
-
|
|
4054
|
+
// A confirmation names the fact the caller read; one that is no longer the
|
|
4055
|
+
// subject's stored fact (replaced, evicted) is the caller's stale view.
|
|
4056
|
+
const current = new Map(source.facts.map((fact) => [fact.subject, fact.observationId]));
|
|
4057
|
+
const outdated = options.confirmed.filter(({ observationId, subject }) => current.get(subject) !== observationId);
|
|
4058
|
+
if (outdated.length > 0) {
|
|
4059
|
+
return yield* new WorkStoreError({ cause: outdated, operation: "reconcile.confirmed" });
|
|
4060
|
+
}
|
|
4061
|
+
// A goal is stamped by the reconciler at most once, whatever its facts
|
|
4062
|
+
// look like later: a reopened goal stays the owner's.
|
|
4063
|
+
const stamped = new Map(source.reconcilerEvents.map(({ eventId, goalId }) => [goalId, eventId]));
|
|
4064
|
+
const cryptoService = this.#cryptoService;
|
|
4065
|
+
const now = yield* Clock.currentTimeMillis;
|
|
4066
|
+
const appendAt = (checkpoint, guard) => this.appendAt(checkpoint, guard);
|
|
4067
|
+
return yield* Effect.forEach(terminalCandidates(source.events, source.facts, source.failures, new Set(options.confirmed.map(({ observationId, subject }) => `${subject}\u0000${observationId}`))), Effect.fnUntraced(function* (candidate) {
|
|
4068
|
+
const goalId = candidate.head.goal.id;
|
|
4069
|
+
const previous = stamped.get(goalId);
|
|
4070
|
+
if (previous !== undefined)
|
|
4071
|
+
return { _tag: "recorded", eventId: previous, goalId };
|
|
4072
|
+
const digest = yield* cryptoService.digest("SHA-256", utf8.encode(`${candidate.fact.observationId}\u0000${goalId}`)).pipe(Effect.mapError(storeError("reconcile.digest")));
|
|
4073
|
+
const eventId = `${reconcilerEventPrefix}${Hex.encode(digest)}`;
|
|
4074
|
+
const checkpoint = terminalCheckpoint(candidate, eventId, now);
|
|
4075
|
+
// A goal that cannot take its terminal checkpoint (an activity already
|
|
4076
|
+
// holds the reconciler's id, a timestamp past the bound) is left for
|
|
4077
|
+
// its owner: reported as a conflict, never failing the other goals.
|
|
4078
|
+
if (Result.isFailure(checkpoint))
|
|
4079
|
+
return { _tag: "conflict", goalId };
|
|
4080
|
+
return yield* appendAt(checkpoint.success, {
|
|
4081
|
+
fact: { observationId: candidate.fact.observationId, subject: candidate.fact.subject },
|
|
4082
|
+
head: candidate.head,
|
|
4083
|
+
reserve: workReconcilerHeadroom
|
|
4084
|
+
}).pipe(Effect.as({
|
|
4085
|
+
_tag: "applied",
|
|
4086
|
+
eventId,
|
|
4087
|
+
goalId,
|
|
4088
|
+
state: candidate.pullRequest.state === "merged" ? "completed" : "abandoned"
|
|
4089
|
+
}), Effect.catchTag("WorkGoalRevisionConflictError", () => Effect.succeed({ _tag: "conflict", goalId })));
|
|
4090
|
+
}));
|
|
4091
|
+
});
|
|
4092
|
+
observe = Effect.fn("WorkStore.observe")(function* (envelopes) {
|
|
4093
|
+
const decoded = yield* Schema.decodeUnknownEffect(Schema.Array(WorkObservationEnvelope).check(Schema.isMaxLength(workObservedFactMaxRecords)))(envelopes).pipe(Effect.mapError(storeError("observe.decode")));
|
|
4094
|
+
const cryptoService = this.#cryptoService;
|
|
4095
|
+
// An observation from further ahead than clock skew allows would outrank
|
|
4096
|
+
// every real reading until wall time caught up, so it is skipped as stale.
|
|
4097
|
+
const clockNow = yield* Clock.currentTimeMillis;
|
|
4098
|
+
const latestAllowed = clockNow + workObservationMaxSkewMillis;
|
|
4099
|
+
const prepared = yield* Effect.forEach(decoded, Effect.fnUntraced(function* (envelope) {
|
|
4100
|
+
// Composite subjects are bounded on their own; check before anything is written.
|
|
4101
|
+
const subject = yield* Schema.decodeUnknownEffect(WorkObservationSubject)(observationSubject(envelope.observation)).pipe(Effect.mapError(storeError("observe.subject")));
|
|
4102
|
+
if (envelope.observedAt > latestAllowed)
|
|
4103
|
+
return { _tag: "future", subject };
|
|
4104
|
+
if (envelope.observation._tag === "unknown") {
|
|
4105
|
+
if (canonicalSubject(envelope.observation.source, envelope.observation.subject) === null) {
|
|
4106
|
+
return yield* new WorkStoreError({ cause: envelope.observation, operation: "observe.subject" });
|
|
4107
|
+
}
|
|
4108
|
+
return {
|
|
4109
|
+
_tag: "unknown",
|
|
4110
|
+
// Within the skew allowance, a reading from slightly ahead counts
|
|
4111
|
+
// as now, so it is visible to a snapshot taken now.
|
|
4112
|
+
observedAt: Math.min(envelope.observedAt, clockNow),
|
|
4113
|
+
reason: envelope.observation.reason,
|
|
4114
|
+
source: envelope.observation.source,
|
|
4115
|
+
subject
|
|
4116
|
+
};
|
|
4117
|
+
}
|
|
4118
|
+
// Case-insensitive identities are lowercased so a spelling change is the
|
|
4119
|
+
// same fact; encoding emits keys in schema field order, so equal facts
|
|
4120
|
+
// give equal text and one id.
|
|
4121
|
+
const record = JSON.stringify(Schema.encodeSync(WorkObservationEnvelope)({
|
|
4122
|
+
...envelope,
|
|
4123
|
+
observation: canonicalObservation(envelope.observation)
|
|
4124
|
+
})
|
|
4125
|
+
.observation);
|
|
4126
|
+
const observationId = yield* cryptoService.digest("SHA-256", utf8.encode(record)).pipe(Effect.map(Hex.encode), Effect.flatMap(Schema.decodeUnknownEffect(WorkObservationId)), Effect.mapError(storeError("observe.digest")));
|
|
4127
|
+
return {
|
|
4128
|
+
_tag: "fact",
|
|
4129
|
+
observationId,
|
|
4130
|
+
observedAt: Math.min(envelope.observedAt, clockNow),
|
|
4131
|
+
record,
|
|
4132
|
+
subject
|
|
4133
|
+
};
|
|
4134
|
+
}));
|
|
4135
|
+
yield* this.secureFiles();
|
|
4136
|
+
const report = yield* Effect.try({
|
|
4137
|
+
try: () => writeObservations(this.#database, prepared),
|
|
4138
|
+
catch: storeError("observe.write")
|
|
4139
|
+
});
|
|
4140
|
+
// The write may have created the WAL siblings; they get the same private mode.
|
|
4141
|
+
yield* this.secureFiles();
|
|
4142
|
+
return report;
|
|
3412
4143
|
});
|
|
3413
4144
|
secureFiles() {
|
|
3414
4145
|
return this.#secureFiles;
|