@nurix/nustack 0.24.0 → 0.25.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/desktopAgent.js +1 -1
- package/dist/commands/doctor.js +8 -11
- package/dist/commands/obligationList.js +23 -49
- package/dist/commands/records.js +6 -88
- package/dist/commands/store.js +199 -0
- package/dist/commands/sync.js +2 -2
- package/dist/commands/toolkit.js +2 -2
- package/dist/lib/context/register.js +1 -1
- package/dist/lib/desktop-agents/checkout.js +1 -1
- package/dist/lib/desktop-agents/claudeRun.js +4 -4
- package/dist/lib/etna/reconcile.js +4 -4
- package/dist/lib/graph/library.js +1 -1
- package/dist/lib/harness/hookCheck.js +2 -8
- package/dist/lib/harness/storeWaivers.js +1 -4
- package/dist/lib/local_store/database.js +12 -92
- package/dist/lib/local_store/device_state.js +2 -2
- package/dist/lib/local_store/index.js +2 -2
- package/dist/lib/local_store/paths.js +0 -6
- package/dist/lib/local_store/routing.js +8 -44
- package/dist/lib/onboarding/folder_state.js +2 -2
- package/dist/lib/onboarding/init_fresh_folder.js +1 -1
- package/dist/lib/onboarding/manifest_writer.js +6 -6
- package/dist/lib/origin_guard.js +1 -1
- package/dist/lib/project_context.js +5 -5
- package/dist/lib/project_contract.js +2 -2
- package/dist/lib/records/drain.js +3 -153
- package/dist/lib/records/kinds.js +16 -0
- package/dist/lib/records/recordInput.js +1 -1
- package/dist/lib/records/recordsTable.js +119 -161
- package/dist/lib/records/refusalLog.js +7 -7
- package/dist/lib/sessions/brief.js +1 -1
- package/dist/lib/sessions/sessionEvents.js +5 -49
- package/dist/lib/store/open.js +136 -0
- package/dist/lib/store/wipe.js +34 -0
- package/dist/lib/sync/apply.js +113 -0
- package/dist/lib/sync/client.js +327 -0
- package/dist/lib/sync/endpoint.js +26 -0
- package/dist/lib/sync/manifest.js +80 -0
- package/dist/lib/sync/outbox.js +104 -0
- package/dist/machine.sql +934 -0
- package/dist/program.js +8 -6
- package/dist/records.schema.json +4 -0
- package/dist/spec_schema.json +5 -5
- package/dist/vendor/harness/rules/docsRoot.js +2 -2
- package/dist/vendor/harness/rules/index.js +1 -1
- package/package.json +5 -5
- package/dist/lib/local_store/migrations.js +0 -399
- package/dist/lib/records/door.js +0 -79
|
@@ -1,33 +1,33 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { withTransaction } from "../local_store/database.js";
|
|
3
|
-
import { readMountDoor } from "./door.js";
|
|
4
3
|
import { AGENT_DEFINITION_ID_VARIABLE, readVariable } from "./environment.js";
|
|
5
4
|
export const SYNC_STATES = ["pending", "synced", "refused", "local"];
|
|
6
5
|
export const PLACEMENTS = ["device", "server"];
|
|
7
6
|
export const CREDENTIALS = ["device"];
|
|
8
|
-
export const SYNCED_RETENTION_DAYS = 30;
|
|
9
7
|
export const RECORDS_LIST_LIMIT = 200;
|
|
10
8
|
export const OBLIGATION_AGENT_OWNED = "OBLIGATION_AGENT_OWNED";
|
|
9
|
+
export const UNBOUND_PROJECT_PREFIX = "unbound:";
|
|
10
|
+
export function unboundProjectId(root) {
|
|
11
|
+
return `${UNBOUND_PROJECT_PREFIX}${root}`;
|
|
12
|
+
}
|
|
13
|
+
export function rootOfUnbound(projectId) {
|
|
14
|
+
return projectId.startsWith(UNBOUND_PROJECT_PREFIX) ? projectId.slice(UNBOUND_PROJECT_PREFIX.length) : null;
|
|
15
|
+
}
|
|
11
16
|
function recordOf(row) {
|
|
17
|
+
const occurredAt = row.occurred_at;
|
|
12
18
|
return {
|
|
13
19
|
id: row.id,
|
|
14
20
|
kind: row.kind,
|
|
15
21
|
schemaVersion: row.schema_version,
|
|
16
|
-
projectId: row.project_id
|
|
22
|
+
projectId: row.project_id,
|
|
17
23
|
sessionId: row.session_id ?? null,
|
|
18
24
|
planAnchor: row.plan_anchor ?? null,
|
|
19
25
|
naturalKey: row.natural_key,
|
|
20
26
|
idempotencyKey: row.idempotency_key,
|
|
21
27
|
payloadJson: row.payload_json,
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
lastError: row.last_error ?? null,
|
|
26
|
-
createdAt: row.created_at,
|
|
27
|
-
repositoryRoot: row.repository_root ?? null,
|
|
28
|
-
doorUrl: row.door_url ?? null,
|
|
29
|
-
placement: row.placement ?? null,
|
|
30
|
-
credential: row.credential ?? null,
|
|
28
|
+
writtenByKind: row.written_by_kind,
|
|
29
|
+
occurredAt,
|
|
30
|
+
createdAt: Date.parse(occurredAt),
|
|
31
31
|
};
|
|
32
32
|
}
|
|
33
33
|
export function canonicalJson(value) {
|
|
@@ -50,6 +50,13 @@ export function syncStateFor(kind, payload, placement) {
|
|
|
50
50
|
return "local";
|
|
51
51
|
return "pending";
|
|
52
52
|
}
|
|
53
|
+
export function placementStateName(id) {
|
|
54
|
+
return `placement:${id}`;
|
|
55
|
+
}
|
|
56
|
+
export function storedPlacementOf(db, id) {
|
|
57
|
+
return db.prepare("SELECT 1 FROM sync_state WHERE name = ?").get(placementStateName(id)) !== undefined ? "device" : "server";
|
|
58
|
+
}
|
|
59
|
+
export const REFUSED_ERROR_PREFIX = "refused: ";
|
|
53
60
|
export class AgentOwnedError extends Error {
|
|
54
61
|
ownerAgentDefinitionId;
|
|
55
62
|
naturalKey;
|
|
@@ -61,7 +68,7 @@ export class AgentOwnedError extends Error {
|
|
|
61
68
|
}
|
|
62
69
|
}
|
|
63
70
|
export function findRecordByKey(db, idempotencyKey) {
|
|
64
|
-
const row = db.prepare("SELECT * FROM
|
|
71
|
+
const row = db.prepare("SELECT * FROM harness_records WHERE idempotency_key = ?").get(idempotencyKey);
|
|
65
72
|
return row ? recordOf(row) : null;
|
|
66
73
|
}
|
|
67
74
|
function checkAgentOwned(stored, write, env) {
|
|
@@ -77,190 +84,134 @@ function checkAgentOwned(stored, write, env) {
|
|
|
77
84
|
return;
|
|
78
85
|
throw new AgentOwnedError(write.naturalKey, owner);
|
|
79
86
|
}
|
|
80
|
-
function
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
87
|
+
function writtenByKindOf(write) {
|
|
88
|
+
return write.credential === "device" ? "agent" : "person";
|
|
89
|
+
}
|
|
90
|
+
function upsert(db, write) {
|
|
91
|
+
const row = db
|
|
92
|
+
.prepare(`INSERT INTO harness_records (id, project_id, kind, schema_version, natural_key, idempotency_key, session_id,
|
|
93
|
+
plan_anchor, payload_json, written_by_kind, occurred_at, created_at, updated_at)
|
|
94
|
+
VALUES (:id, :project_id, :kind, :schema_version, :natural_key, :idempotency_key, :session_id,
|
|
95
|
+
:plan_anchor, :payload_json, :written_by_kind, :occurred_at, datetime('now'), datetime('now'))
|
|
96
|
+
ON CONFLICT (project_id, idempotency_key) DO UPDATE SET
|
|
97
|
+
kind = excluded.kind, schema_version = excluded.schema_version, natural_key = excluded.natural_key,
|
|
98
|
+
session_id = excluded.session_id, plan_anchor = excluded.plan_anchor, payload_json = excluded.payload_json,
|
|
99
|
+
written_by_kind = excluded.written_by_kind, occurred_at = excluded.occurred_at, updated_at = datetime('now')
|
|
100
|
+
RETURNING id`)
|
|
101
|
+
.get({
|
|
93
102
|
id: randomUUID(),
|
|
103
|
+
project_id: write.projectId ?? unboundProjectId(write.repositoryRoot ?? ""),
|
|
94
104
|
kind: write.kind,
|
|
95
105
|
schema_version: write.schemaVersion,
|
|
96
|
-
project_id: write.projectId,
|
|
97
|
-
session_id: write.sessionId,
|
|
98
|
-
plan_anchor: write.planAnchor,
|
|
99
106
|
natural_key: write.naturalKey,
|
|
100
107
|
idempotency_key: write.idempotencyKey,
|
|
108
|
+
session_id: write.sessionId,
|
|
109
|
+
plan_anchor: write.planAnchor,
|
|
101
110
|
payload_json: JSON.stringify(write.payload),
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
repository_root: write.repositoryRoot,
|
|
105
|
-
placement,
|
|
106
|
-
credential: write.credential,
|
|
111
|
+
written_by_kind: writtenByKindOf(write),
|
|
112
|
+
occurred_at: new Date(write.createdAt).toISOString(),
|
|
107
113
|
});
|
|
114
|
+
return row.id;
|
|
108
115
|
}
|
|
109
116
|
export function writeRecords(store, writes, env = process.env) {
|
|
110
117
|
return withTransaction(store, (db) => writes.map((write) => {
|
|
111
118
|
const stored = findRecordByKey(db, write.idempotencyKey);
|
|
112
119
|
checkAgentOwned(stored, write, env);
|
|
113
|
-
const
|
|
120
|
+
const id = upsert(db, write);
|
|
121
|
+
const placement = stored ? storedPlacementOf(db, id) : write.placement;
|
|
122
|
+
if (!stored && placement === "device")
|
|
123
|
+
db.prepare("INSERT OR REPLACE INTO sync_state (name, value) VALUES (?, 'device')").run(placementStateName(id));
|
|
114
124
|
const syncState = syncStateFor(write.kind, write.payload, placement);
|
|
115
|
-
|
|
125
|
+
if (syncState === "local")
|
|
126
|
+
db.prepare("DELETE FROM sync_outbox WHERE table_name = 'harness_records' AND row_key = ?").run(id);
|
|
116
127
|
return { idempotencyKey: write.idempotencyKey, syncState };
|
|
117
128
|
}));
|
|
118
129
|
}
|
|
119
130
|
export function releaseUnboundRows(store, root) {
|
|
120
131
|
store.db
|
|
121
|
-
.prepare(`UPDATE
|
|
122
|
-
WHERE
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
export function selectDueRecords(store, now, canSendDevice) {
|
|
126
|
-
const rows = store.db
|
|
127
|
-
.prepare(`SELECT * FROM records
|
|
128
|
-
WHERE sync_state = 'pending' AND (next_attempt_at IS NULL OR next_attempt_at <= ?)
|
|
129
|
-
AND (credential IS NULL OR ?)
|
|
130
|
-
ORDER BY created_at ASC, rowid ASC`)
|
|
131
|
-
.all(now, canSendDevice ? 1 : 0);
|
|
132
|
-
return rows.map(recordOf);
|
|
132
|
+
.prepare(`UPDATE sync_outbox SET next_attempt_at = NULL
|
|
133
|
+
WHERE table_name = 'harness_records' AND next_attempt_at IS NOT NULL
|
|
134
|
+
AND row_key IN (SELECT id FROM harness_records WHERE project_id = ?)`)
|
|
135
|
+
.run(unboundProjectId(root));
|
|
133
136
|
}
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
WHERE sync_state = 'pending' AND (next_attempt_at IS NULL OR next_attempt_at <= ?)
|
|
138
|
-
AND (credential IS NULL OR ?) LIMIT 1`)
|
|
139
|
-
.get(now, canSendDevice ? 1 : 0) !== undefined);
|
|
140
|
-
}
|
|
141
|
-
const CAS = `id = :id AND sync_state = 'pending' AND payload_json = :payload_json AND session_id IS :session_id
|
|
142
|
-
AND plan_anchor IS :plan_anchor AND created_at = :created_at`;
|
|
143
|
-
function casParameters(row) {
|
|
144
|
-
return {
|
|
145
|
-
id: row.id,
|
|
146
|
-
payload_json: row.payloadJson,
|
|
147
|
-
session_id: row.sessionId,
|
|
148
|
-
plan_anchor: row.planAnchor,
|
|
149
|
-
created_at: row.createdAt,
|
|
150
|
-
};
|
|
151
|
-
}
|
|
152
|
-
export function markSynced(store, row) {
|
|
153
|
-
const result = store.db
|
|
154
|
-
.prepare(`UPDATE records SET sync_state = 'synced', last_error = NULL, next_attempt_at = NULL WHERE ${CAS}`)
|
|
155
|
-
.run(casParameters(row));
|
|
156
|
-
return Number(result.changes) > 0;
|
|
157
|
-
}
|
|
158
|
-
export function markRefused(store, row, error) {
|
|
159
|
-
const result = store.db
|
|
160
|
-
.prepare(`UPDATE records SET sync_state = 'refused', last_error = :last_error, next_attempt_at = NULL WHERE ${CAS}`)
|
|
161
|
-
.run({ ...casParameters(row), last_error: error });
|
|
162
|
-
return Number(result.changes) > 0;
|
|
163
|
-
}
|
|
164
|
-
export function markBackoff(store, row, error, nextAttemptAt) {
|
|
165
|
-
const result = store.db
|
|
166
|
-
.prepare(`UPDATE records SET attempts = attempts + 1, next_attempt_at = :next_attempt_at, last_error = :last_error WHERE ${CAS}`)
|
|
167
|
-
.run({ ...casParameters(row), last_error: error, next_attempt_at: nextAttemptAt });
|
|
168
|
-
return Number(result.changes) > 0;
|
|
169
|
-
}
|
|
170
|
-
export function markDeferred(store, row, nextAttemptAt) {
|
|
171
|
-
const result = store.db
|
|
172
|
-
.prepare(`UPDATE records SET next_attempt_at = :next_attempt_at WHERE ${CAS}`)
|
|
173
|
-
.run({ ...casParameters(row), next_attempt_at: nextAttemptAt });
|
|
174
|
-
return Number(result.changes) > 0;
|
|
137
|
+
function dropUnpushed(db, id) {
|
|
138
|
+
db.prepare("DELETE FROM harness_records WHERE id = ?").run(id);
|
|
139
|
+
db.prepare("DELETE FROM sync_outbox WHERE table_name = 'harness_records' AND row_key = ?").run(id);
|
|
175
140
|
}
|
|
176
141
|
export function rekeyRecord(store, row, projectId) {
|
|
177
142
|
const key = `${projectId}:${row.kind}:${row.naturalKey}`;
|
|
178
143
|
return withTransaction(store, (db) => {
|
|
179
|
-
const current = db.prepare("SELECT * FROM
|
|
144
|
+
const current = db.prepare("SELECT * FROM harness_records WHERE id = ?").get(row.id);
|
|
180
145
|
if (!current)
|
|
181
146
|
return null;
|
|
182
|
-
const existing =
|
|
183
|
-
if (existing
|
|
184
|
-
|
|
185
|
-
|
|
147
|
+
const existing = db.prepare("SELECT * FROM harness_records WHERE project_id = ? AND idempotency_key = ?").get(projectId, key);
|
|
148
|
+
if (existing) {
|
|
149
|
+
const stored = recordOf(existing);
|
|
150
|
+
dropUnpushed(db, row.id);
|
|
151
|
+
if (stored.createdAt >= row.createdAt)
|
|
186
152
|
return null;
|
|
187
|
-
|
|
188
|
-
|
|
153
|
+
db.prepare(`UPDATE harness_records SET kind = ?, schema_version = ?, natural_key = ?, session_id = ?, plan_anchor = ?,
|
|
154
|
+
payload_json = ?, written_by_kind = ?, occurred_at = ?, updated_at = datetime('now') WHERE id = ?`).run(row.kind, row.schemaVersion, row.naturalKey, row.sessionId, row.planAnchor, row.payloadJson, row.writtenByKind, row.occurredAt, stored.id);
|
|
155
|
+
return recordOf(db.prepare("SELECT * FROM harness_records WHERE id = ?").get(stored.id));
|
|
189
156
|
}
|
|
190
|
-
db.prepare("UPDATE
|
|
191
|
-
return recordOf(db.prepare("SELECT * FROM
|
|
157
|
+
db.prepare("UPDATE harness_records SET project_id = ?, idempotency_key = ?, updated_at = datetime('now') WHERE id = ?").run(projectId, key, row.id);
|
|
158
|
+
return recordOf(db.prepare("SELECT * FROM harness_records WHERE id = ?").get(row.id));
|
|
192
159
|
});
|
|
193
160
|
}
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
AND NOT (kind = 'obligation' AND json_extract(payload_json, '$.state') = 'open')`)
|
|
198
|
-
.run(now - SYNCED_RETENTION_DAYS * 24 * 60 * 60 * 1000);
|
|
199
|
-
}
|
|
161
|
+
const VIEW_SELECT = `SELECT r.*, o.row_key AS outbox_key, o.attempts AS outbox_attempts, o.next_attempt_at AS outbox_next_attempt_at,
|
|
162
|
+
o.last_error AS outbox_last_error
|
|
163
|
+
FROM harness_records r LEFT JOIN sync_outbox o ON o.table_name = 'harness_records' AND o.row_key = r.id`;
|
|
200
164
|
function viewOf(row) {
|
|
165
|
+
const stored = recordOf(row);
|
|
201
166
|
let payload;
|
|
202
167
|
try {
|
|
203
|
-
payload = JSON.parse(
|
|
168
|
+
payload = JSON.parse(stored.payloadJson);
|
|
204
169
|
}
|
|
205
170
|
catch {
|
|
206
|
-
payload =
|
|
171
|
+
payload = stored.payloadJson;
|
|
207
172
|
}
|
|
173
|
+
const lastError = row.outbox_last_error ?? null;
|
|
174
|
+
const syncState = row.outbox_key !== null && row.outbox_key !== undefined
|
|
175
|
+
? lastError?.startsWith(REFUSED_ERROR_PREFIX)
|
|
176
|
+
? "refused"
|
|
177
|
+
: "pending"
|
|
178
|
+
: row.sync_seq !== null
|
|
179
|
+
? "synced"
|
|
180
|
+
: "local";
|
|
181
|
+
const root = rootOfUnbound(stored.projectId);
|
|
208
182
|
return {
|
|
209
|
-
id:
|
|
210
|
-
kind:
|
|
211
|
-
projectId:
|
|
212
|
-
sessionId:
|
|
213
|
-
planAnchor:
|
|
214
|
-
naturalKey:
|
|
215
|
-
idempotencyKey:
|
|
183
|
+
id: stored.id,
|
|
184
|
+
kind: stored.kind,
|
|
185
|
+
projectId: root === null ? stored.projectId : null,
|
|
186
|
+
sessionId: stored.sessionId,
|
|
187
|
+
planAnchor: stored.planAnchor,
|
|
188
|
+
naturalKey: stored.naturalKey,
|
|
189
|
+
idempotencyKey: stored.idempotencyKey,
|
|
216
190
|
payload,
|
|
217
|
-
syncState
|
|
218
|
-
attempts: row.
|
|
219
|
-
nextAttemptAt: row.
|
|
220
|
-
lastError
|
|
221
|
-
createdAt:
|
|
222
|
-
repositoryRoot:
|
|
223
|
-
|
|
224
|
-
credential: row.credential,
|
|
191
|
+
syncState,
|
|
192
|
+
attempts: Number(row.outbox_attempts ?? 0),
|
|
193
|
+
nextAttemptAt: row.outbox_next_attempt_at ?? null,
|
|
194
|
+
lastError,
|
|
195
|
+
createdAt: stored.createdAt,
|
|
196
|
+
repositoryRoot: root,
|
|
197
|
+
credential: stored.writtenByKind === "agent" ? "device" : null,
|
|
225
198
|
};
|
|
226
199
|
}
|
|
227
|
-
export
|
|
228
|
-
const
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
const
|
|
232
|
-
const
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
continue;
|
|
236
|
-
if (!doorOff.has(row.repositoryRoot))
|
|
237
|
-
doorOff.set(row.repositoryRoot, (await readMountDoor(row.repositoryRoot)) === null);
|
|
238
|
-
}
|
|
239
|
-
const isDoorOff = (row) => row.doorUrl === null && row.repositoryRoot !== null && doorOff.get(row.repositoryRoot) === true;
|
|
240
|
-
let waitingCount = 0;
|
|
241
|
-
let refusedWithProject = 0;
|
|
242
|
-
let storeCount = 0;
|
|
243
|
-
for (const row of records) {
|
|
244
|
-
if (row.syncState === "refused" && row.projectId !== null)
|
|
245
|
-
refusedWithProject += 1;
|
|
246
|
-
if (row.syncState !== "pending")
|
|
247
|
-
continue;
|
|
248
|
-
if (row.projectId === null || isDoorOff(row))
|
|
249
|
-
storeCount += 1;
|
|
250
|
-
else
|
|
251
|
-
waitingCount += 1;
|
|
252
|
-
}
|
|
253
|
-
const outboxCount = waitingCount + refusedWithProject;
|
|
254
|
-
const newest = records
|
|
255
|
-
.filter((row) => (row.syncState === "pending" || row.syncState === "refused") && row.lastError !== null)
|
|
256
|
-
.sort((a, b) => b.createdAt - a.createdAt)[0];
|
|
200
|
+
export function countRecords(store, projectId) {
|
|
201
|
+
const where = `WHERE (o.row_key IS NOT NULL OR r.sync_seq IS NULL)${projectId ? " AND r.project_id = ?" : ""}`;
|
|
202
|
+
const rows = (projectId ? store.db.prepare(`${VIEW_SELECT} ${where}`).all(projectId) : store.db.prepare(`${VIEW_SELECT} ${where}`).all());
|
|
203
|
+
const views = rows.map(viewOf);
|
|
204
|
+
const isBound = (view) => view.repositoryRoot === null;
|
|
205
|
+
const waitingCount = views.filter((view) => view.syncState === "pending" && isBound(view)).length;
|
|
206
|
+
const refusedWithProject = views.filter((view) => view.syncState === "refused" && isBound(view)).length;
|
|
207
|
+
const newest = views.filter((view) => view.lastError !== null).sort((a, b) => b.createdAt - a.createdAt)[0];
|
|
257
208
|
return {
|
|
258
|
-
outboxCount,
|
|
259
|
-
storeCount,
|
|
209
|
+
outboxCount: waitingCount + refusedWithProject,
|
|
210
|
+
storeCount: views.filter((view) => view.syncState === "pending" && !isBound(view)).length,
|
|
260
211
|
lastError: newest?.lastError ?? null,
|
|
261
|
-
pendingCount:
|
|
262
|
-
refusedCount:
|
|
263
|
-
localCount:
|
|
212
|
+
pendingCount: views.filter((view) => view.syncState === "pending").length,
|
|
213
|
+
refusedCount: views.filter((view) => view.syncState === "refused").length,
|
|
214
|
+
localCount: views.filter((view) => view.syncState === "local").length,
|
|
264
215
|
waitingCount,
|
|
265
216
|
};
|
|
266
217
|
}
|
|
@@ -268,20 +219,27 @@ export function listRecords(store, filter = {}) {
|
|
|
268
219
|
const clauses = [];
|
|
269
220
|
const parameters = [];
|
|
270
221
|
if (filter.projectId) {
|
|
271
|
-
clauses.push("project_id = ?");
|
|
222
|
+
clauses.push("r.project_id = ?");
|
|
272
223
|
parameters.push(filter.projectId);
|
|
273
224
|
}
|
|
274
225
|
if (filter.kind) {
|
|
275
|
-
clauses.push("kind = ?");
|
|
226
|
+
clauses.push("r.kind = ?");
|
|
276
227
|
parameters.push(filter.kind);
|
|
277
228
|
}
|
|
278
229
|
if (filter.planAnchor) {
|
|
279
|
-
clauses.push("plan_anchor = ?");
|
|
230
|
+
clauses.push("r.plan_anchor = ?");
|
|
280
231
|
parameters.push(filter.planAnchor);
|
|
281
232
|
}
|
|
282
233
|
const where = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : "";
|
|
283
234
|
const rows = store.db
|
|
284
|
-
.prepare(
|
|
235
|
+
.prepare(`${VIEW_SELECT} ${where} ORDER BY r.occurred_at DESC, r.rowid DESC LIMIT ${RECORDS_LIST_LIMIT}`)
|
|
285
236
|
.all(...parameters);
|
|
286
|
-
return rows.map(
|
|
237
|
+
return rows.map(viewOf);
|
|
238
|
+
}
|
|
239
|
+
export function selectUnboundRecords(db) {
|
|
240
|
+
const rows = db
|
|
241
|
+
.prepare(`SELECT r.* FROM harness_records r JOIN sync_outbox o ON o.table_name = 'harness_records' AND o.row_key = r.id
|
|
242
|
+
WHERE r.project_id LIKE '${UNBOUND_PROJECT_PREFIX}%' ORDER BY r.occurred_at, r.rowid`)
|
|
243
|
+
.all();
|
|
244
|
+
return rows.map(recordOf);
|
|
287
245
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { RECORD_SCHEMA_VERSION } from "./
|
|
2
|
+
import { RECORD_SCHEMA_VERSION } from "./kinds.js";
|
|
3
3
|
import { idempotencyKeyOf } from "./naturalKey.js";
|
|
4
|
-
import { writeRecords } from "./recordsTable.js";
|
|
4
|
+
import { UNBOUND_PROJECT_PREFIX, writeRecords } from "./recordsTable.js";
|
|
5
5
|
export const REFUSAL_GATE_IDS = [
|
|
6
6
|
"records-validation",
|
|
7
7
|
"records-unknown-kind",
|
|
@@ -69,19 +69,19 @@ export function countRefusals(store, filter) {
|
|
|
69
69
|
const ids = REFUSAL_GATE_IDS.map(() => "?").join(", ");
|
|
70
70
|
const clauses = [
|
|
71
71
|
"kind = 'gate_event'",
|
|
72
|
-
"project_id
|
|
72
|
+
filter.projectId === null ? `project_id LIKE '${UNBOUND_PROJECT_PREFIX}%'` : "project_id = ?",
|
|
73
73
|
`json_extract(payload_json, '$.gate') IN (${ids})`,
|
|
74
74
|
"json_extract(payload_json, '$.verdict') = ?",
|
|
75
75
|
];
|
|
76
|
-
const parameters = [filter.projectId, ...REFUSAL_GATE_IDS, filter.verdict ?? "block"];
|
|
76
|
+
const parameters = [...(filter.projectId === null ? [] : [filter.projectId]), ...REFUSAL_GATE_IDS, filter.verdict ?? "block"];
|
|
77
77
|
if (filter.sessionId !== undefined) {
|
|
78
78
|
clauses.push("session_id = ?");
|
|
79
79
|
parameters.push(filter.sessionId);
|
|
80
80
|
}
|
|
81
81
|
if (filter.since !== undefined) {
|
|
82
|
-
clauses.push("
|
|
83
|
-
parameters.push(filter.since);
|
|
82
|
+
clauses.push("occurred_at >= ?");
|
|
83
|
+
parameters.push(new Date(filter.since).toISOString());
|
|
84
84
|
}
|
|
85
|
-
const row = store.db.prepare(`SELECT count(*) AS n FROM
|
|
85
|
+
const row = store.db.prepare(`SELECT count(*) AS n FROM harness_records WHERE ${clauses.join(" AND ")}`).get(...parameters);
|
|
86
86
|
return Number(row.n);
|
|
87
87
|
}
|
|
@@ -3,7 +3,7 @@ import path from "node:path";
|
|
|
3
3
|
import { ensureAccessToken } from "../auth_grant.js";
|
|
4
4
|
import { briefCachePath } from "../local_store/paths.js";
|
|
5
5
|
import { assertCredentialOrigin } from "../nustack_client.js";
|
|
6
|
-
import { resolveDoor } from "../
|
|
6
|
+
import { resolveDoor } from "../sync/endpoint.js";
|
|
7
7
|
export const BRIEF_FETCH_TIMEOUT_MS = 3_000;
|
|
8
8
|
export const BRIEF_CONTEXT_CAP = 4_000;
|
|
9
9
|
export const BRIEF_CONTEXT_OBLIGATIONS = 20;
|
|
@@ -2,8 +2,8 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, unlinkSyn
|
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { ensureAccessToken } from "../auth_grant.js";
|
|
4
4
|
import { sessionEventsDir } from "../local_store/paths.js";
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
5
|
+
import { RECORDS_BATCH_CAP } from "../records/kinds.js";
|
|
6
|
+
import { resolveDoor } from "../sync/endpoint.js";
|
|
7
7
|
import { makeSessionsClient, postSessionEvents } from "./client.js";
|
|
8
8
|
export const SESSION_SPOOL_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
|
|
9
9
|
const SESSION_DRAIN_PASSES = 3;
|
|
@@ -120,52 +120,9 @@ export async function drainSessionEvents(globals, result, bearers = new Map()) {
|
|
|
120
120
|
}
|
|
121
121
|
}
|
|
122
122
|
}
|
|
123
|
-
function spoolOf(sessionId, spools) {
|
|
123
|
+
export function spoolOf(sessionId, spools) {
|
|
124
124
|
return spools.find((spool) => spool.rowId === sessionId) ?? spools.find((spool) => spool.runtimeSessionId === sessionId) ?? null;
|
|
125
125
|
}
|
|
126
|
-
export async function drainLabelRows(store, globals, result, bearers = new Map(), now = Date.now()) {
|
|
127
|
-
const rows = listPendingLabelRows(store, RECORDS_BATCH_CAP);
|
|
128
|
-
if (rows.length === 0)
|
|
129
|
-
return;
|
|
130
|
-
const spools = listSpools();
|
|
131
|
-
const groups = new Map();
|
|
132
|
-
const stale = [];
|
|
133
|
-
for (const record of rows) {
|
|
134
|
-
const spool = spoolOf(record.session_id, spools);
|
|
135
|
-
if (!spool) {
|
|
136
|
-
if (now - record.created_at > SESSION_SPOOL_RETENTION_MS)
|
|
137
|
-
stale.push(record.id);
|
|
138
|
-
continue;
|
|
139
|
-
}
|
|
140
|
-
const key = `${spool.root}\u0000${spool.projectId}`;
|
|
141
|
-
const group = groups.get(key) ?? { spool, rows: [] };
|
|
142
|
-
group.rows.push({ record, wire: { ...labelRowWire(record), sessionId: spool.rowId } });
|
|
143
|
-
groups.set(key, group);
|
|
144
|
-
}
|
|
145
|
-
if (stale.length > 0)
|
|
146
|
-
result.labelRowsRefused += markLabelRowsRefused(store, stale);
|
|
147
|
-
for (const { spool, rows: batch } of groups.values()) {
|
|
148
|
-
const door = await doorFor(spool.root, globals, bearers);
|
|
149
|
-
if (!door)
|
|
150
|
-
continue;
|
|
151
|
-
const request = { events: [], labelRows: batch.map((row) => row.wire) };
|
|
152
|
-
const outcome = await postSessionEvents(door.doorUrl, door.bearer, spool.projectId, request);
|
|
153
|
-
if (outcome.ok) {
|
|
154
|
-
result.labelRowsSynced += markLabelRowsSynced(store, batch.map((row) => row.record.id));
|
|
155
|
-
continue;
|
|
156
|
-
}
|
|
157
|
-
result.lastError = outcome.message;
|
|
158
|
-
if (!isRefusal(outcome))
|
|
159
|
-
continue;
|
|
160
|
-
for (const row of batch) {
|
|
161
|
-
const alone = await postSessionEvents(door.doorUrl, door.bearer, spool.projectId, { events: [], labelRows: [row.wire] });
|
|
162
|
-
if (alone.ok)
|
|
163
|
-
result.labelRowsSynced += markLabelRowsSynced(store, [row.record.id]);
|
|
164
|
-
else if (isRefusal(alone))
|
|
165
|
-
result.labelRowsRefused += markLabelRowsRefused(store, [row.record.id]);
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
126
|
export function pruneSpools(now = Date.now()) {
|
|
170
127
|
let pruned = 0;
|
|
171
128
|
for (const spool of listSpools()) {
|
|
@@ -182,8 +139,8 @@ export function pruneSpools(now = Date.now()) {
|
|
|
182
139
|
}
|
|
183
140
|
return pruned;
|
|
184
141
|
}
|
|
185
|
-
export async function drainSessionLegs(
|
|
186
|
-
const result = { eventsSent: 0, eventsDropped: 0, closed: 0,
|
|
142
|
+
export async function drainSessionLegs(globals) {
|
|
143
|
+
const result = { eventsSent: 0, eventsDropped: 0, closed: 0, lastError: null };
|
|
187
144
|
const bearers = new Map();
|
|
188
145
|
try {
|
|
189
146
|
for (let pass = 0; pass < SESSION_DRAIN_PASSES; pass += 1) {
|
|
@@ -193,7 +150,6 @@ export async function drainSessionLegs(store, globals) {
|
|
|
193
150
|
if (isIdle || result.eventsSent + result.eventsDropped + result.closed === before)
|
|
194
151
|
break;
|
|
195
152
|
}
|
|
196
|
-
await drainLabelRows(store, globals, result, bearers);
|
|
197
153
|
pruneSpools();
|
|
198
154
|
}
|
|
199
155
|
catch (error) {
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { existsSync, readFileSync, rmSync } from "node:fs";
|
|
2
|
+
import { chmod } from "node:fs/promises";
|
|
3
|
+
import os from "node:os";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { mkdirSecure } from "../secure_file.js";
|
|
6
|
+
import { nustackHomeDir } from "../local_store/paths.js";
|
|
7
|
+
import { StoreTooNewError, versionVerdict, wipeStaleStore } from "./wipe.js";
|
|
8
|
+
export function machineStorePath(home = os.homedir()) {
|
|
9
|
+
return path.join(home, ".nustack", "nustack.db");
|
|
10
|
+
}
|
|
11
|
+
export function databaseFiles(dbFile) {
|
|
12
|
+
return [dbFile, `${dbFile}-wal`, `${dbFile}-shm`];
|
|
13
|
+
}
|
|
14
|
+
export function storeStampOf(version) {
|
|
15
|
+
const match = /^v?(\d+)\.(\d+)/.exec(version.trim());
|
|
16
|
+
if (!match)
|
|
17
|
+
throw new Error(`not a release number: ${version}`);
|
|
18
|
+
return Number(match[1]) * 1000 + Number(match[2]);
|
|
19
|
+
}
|
|
20
|
+
let cliVersionCache;
|
|
21
|
+
export function cliVersion() {
|
|
22
|
+
cliVersionCache ??= JSON.parse(readFileSync(new URL("../../../package.json", import.meta.url), "utf8")).version;
|
|
23
|
+
return cliVersionCache;
|
|
24
|
+
}
|
|
25
|
+
let schemaCache;
|
|
26
|
+
function machineSchemaSql() {
|
|
27
|
+
schemaCache ??= readFileSync(new URL("../../../dist/machine.sql", import.meta.url), "utf8");
|
|
28
|
+
return schemaCache;
|
|
29
|
+
}
|
|
30
|
+
let sqliteModulePromise;
|
|
31
|
+
export async function withSuppressedSqliteWarning(fn) {
|
|
32
|
+
const original = process.emitWarning;
|
|
33
|
+
const filtered = (warning, ...args) => {
|
|
34
|
+
const name = warning instanceof Error
|
|
35
|
+
? warning.name
|
|
36
|
+
: typeof args[0] === "string"
|
|
37
|
+
? args[0]
|
|
38
|
+
: args[0]?.type;
|
|
39
|
+
const message = warning instanceof Error ? warning.message : String(warning);
|
|
40
|
+
if (name === "ExperimentalWarning" && /sqlite/i.test(message))
|
|
41
|
+
return;
|
|
42
|
+
original(warning, ...args);
|
|
43
|
+
};
|
|
44
|
+
process.emitWarning = filtered;
|
|
45
|
+
try {
|
|
46
|
+
return await fn();
|
|
47
|
+
}
|
|
48
|
+
finally {
|
|
49
|
+
process.emitWarning = original;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
export async function openSqliteFile(dbFile, options = {}) {
|
|
53
|
+
sqliteModulePromise ??= withSuppressedSqliteWarning(() => import("node:sqlite"));
|
|
54
|
+
const { DatabaseSync } = await sqliteModulePromise;
|
|
55
|
+
return new DatabaseSync(dbFile, options.readOnly ? { readOnly: true } : {});
|
|
56
|
+
}
|
|
57
|
+
function userVersionOf(db) {
|
|
58
|
+
const row = db.prepare("PRAGMA user_version").get();
|
|
59
|
+
return Number(row?.user_version ?? 0);
|
|
60
|
+
}
|
|
61
|
+
function retireReplacedStores() {
|
|
62
|
+
for (const file of databaseFiles(path.join(nustackHomeDir(), "state.db"))) {
|
|
63
|
+
try {
|
|
64
|
+
rmSync(file, { force: true });
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
function settleSchema(db, stamp) {
|
|
71
|
+
if (userVersionOf(db) === stamp)
|
|
72
|
+
return true;
|
|
73
|
+
db.exec("BEGIN IMMEDIATE");
|
|
74
|
+
try {
|
|
75
|
+
const found = userVersionOf(db);
|
|
76
|
+
const tableCount = Number(db.prepare("SELECT count(*) AS n FROM sqlite_master").get().n);
|
|
77
|
+
if (found === 0 && tableCount === 0) {
|
|
78
|
+
db.exec(machineSchemaSql());
|
|
79
|
+
db.exec(`PRAGMA user_version = ${stamp}`);
|
|
80
|
+
db.exec("COMMIT");
|
|
81
|
+
return true;
|
|
82
|
+
}
|
|
83
|
+
db.exec("ROLLBACK");
|
|
84
|
+
const verdict = versionVerdict(found, stamp);
|
|
85
|
+
if (verdict === "same")
|
|
86
|
+
return true;
|
|
87
|
+
if (verdict === "newer")
|
|
88
|
+
throw new StoreTooNewError(found, stamp);
|
|
89
|
+
return false;
|
|
90
|
+
}
|
|
91
|
+
catch (error) {
|
|
92
|
+
try {
|
|
93
|
+
db.exec("ROLLBACK");
|
|
94
|
+
}
|
|
95
|
+
catch {
|
|
96
|
+
}
|
|
97
|
+
throw error;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
export async function openMachineDatabase(dbFile = machineStorePath()) {
|
|
101
|
+
retireReplacedStores();
|
|
102
|
+
await mkdirSecure(path.dirname(dbFile));
|
|
103
|
+
const stamp = storeStampOf(cliVersion());
|
|
104
|
+
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
105
|
+
const db = await withSuppressedSqliteWarning(() => openSqliteFile(dbFile));
|
|
106
|
+
try {
|
|
107
|
+
db.exec("PRAGMA busy_timeout = 5000");
|
|
108
|
+
db.exec("PRAGMA journal_mode = WAL");
|
|
109
|
+
db.exec("PRAGMA foreign_keys = OFF");
|
|
110
|
+
if (settleSchema(db, stamp)) {
|
|
111
|
+
await chmod(dbFile, 0o600).catch(() => { });
|
|
112
|
+
return db;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
catch (error) {
|
|
116
|
+
db.close();
|
|
117
|
+
throw error;
|
|
118
|
+
}
|
|
119
|
+
await wipeStaleStore(db, dbFile);
|
|
120
|
+
}
|
|
121
|
+
throw new Error(`the machine store at ${dbFile} could not be recreated`);
|
|
122
|
+
}
|
|
123
|
+
export async function openMachineDatabaseReadOnly(dbFile = machineStorePath()) {
|
|
124
|
+
if (!existsSync(dbFile))
|
|
125
|
+
return null;
|
|
126
|
+
const db = await withSuppressedSqliteWarning(() => openSqliteFile(dbFile, { readOnly: true }));
|
|
127
|
+
try {
|
|
128
|
+
db.exec("PRAGMA busy_timeout = 2000");
|
|
129
|
+
if (userVersionOf(db) === storeStampOf(cliVersion()))
|
|
130
|
+
return db;
|
|
131
|
+
}
|
|
132
|
+
catch {
|
|
133
|
+
}
|
|
134
|
+
db.close();
|
|
135
|
+
return null;
|
|
136
|
+
}
|