@nurix/nustack 0.23.34 → 0.25.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/dist/commands/desktopAgent.js +1 -1
  2. package/dist/commands/doctor.js +8 -11
  3. package/dist/commands/obligationList.js +23 -49
  4. package/dist/commands/records.js +6 -88
  5. package/dist/commands/store.js +199 -0
  6. package/dist/commands/sync.js +2 -2
  7. package/dist/commands/toolkit.js +2 -2
  8. package/dist/lib/context/register.js +1 -1
  9. package/dist/lib/desktop-agents/checkout.js +1 -1
  10. package/dist/lib/desktop-agents/claudeRun.js +4 -4
  11. package/dist/lib/etna/reconcile.js +4 -4
  12. package/dist/lib/graph/library.js +1 -1
  13. package/dist/lib/harness/hookCheck.js +2 -8
  14. package/dist/lib/harness/storeWaivers.js +1 -4
  15. package/dist/lib/local_store/database.js +12 -92
  16. package/dist/lib/local_store/device_state.js +2 -2
  17. package/dist/lib/local_store/index.js +2 -2
  18. package/dist/lib/local_store/paths.js +0 -6
  19. package/dist/lib/local_store/routing.js +8 -44
  20. package/dist/lib/onboarding/folder_state.js +2 -2
  21. package/dist/lib/onboarding/init_fresh_folder.js +1 -1
  22. package/dist/lib/onboarding/manifest_writer.js +6 -6
  23. package/dist/lib/origin_guard.js +1 -1
  24. package/dist/lib/project_context.js +5 -5
  25. package/dist/lib/project_contract.js +2 -2
  26. package/dist/lib/records/drain.js +3 -153
  27. package/dist/lib/records/kinds.js +16 -0
  28. package/dist/lib/records/recordInput.js +1 -1
  29. package/dist/lib/records/recordsTable.js +119 -161
  30. package/dist/lib/records/refusalLog.js +7 -7
  31. package/dist/lib/sessions/brief.js +1 -1
  32. package/dist/lib/sessions/sessionEvents.js +5 -49
  33. package/dist/lib/store/open.js +136 -0
  34. package/dist/lib/store/wipe.js +34 -0
  35. package/dist/lib/sync/apply.js +113 -0
  36. package/dist/lib/sync/client.js +327 -0
  37. package/dist/lib/sync/endpoint.js +26 -0
  38. package/dist/lib/sync/manifest.js +80 -0
  39. package/dist/lib/sync/outbox.js +104 -0
  40. package/dist/machine.sql +934 -0
  41. package/dist/program.js +8 -6
  42. package/dist/records.schema.json +4 -0
  43. package/dist/spec_schema.json +5 -5
  44. package/dist/vendor/harness/rules/docsRoot.js +2 -2
  45. package/dist/vendor/harness/rules/index.js +1 -1
  46. package/package.json +6 -6
  47. package/dist/lib/local_store/migrations.js +0 -399
  48. package/dist/lib/records/door.js +0 -79
@@ -1,33 +1,33 @@
1
1
  import { randomUUID } from "node:crypto";
2
2
  import { withTransaction } from "../local_store/database.js";
3
- import { readMountDoor } from "./door.js";
4
3
  import { AGENT_DEFINITION_ID_VARIABLE, readVariable } from "./environment.js";
5
4
  export const SYNC_STATES = ["pending", "synced", "refused", "local"];
6
5
  export const PLACEMENTS = ["device", "server"];
7
6
  export const CREDENTIALS = ["device"];
8
- export const SYNCED_RETENTION_DAYS = 30;
9
7
  export const RECORDS_LIST_LIMIT = 200;
10
8
  export const OBLIGATION_AGENT_OWNED = "OBLIGATION_AGENT_OWNED";
9
+ export const UNBOUND_PROJECT_PREFIX = "unbound:";
10
+ export function unboundProjectId(root) {
11
+ return `${UNBOUND_PROJECT_PREFIX}${root}`;
12
+ }
13
+ export function rootOfUnbound(projectId) {
14
+ return projectId.startsWith(UNBOUND_PROJECT_PREFIX) ? projectId.slice(UNBOUND_PROJECT_PREFIX.length) : null;
15
+ }
11
16
  function recordOf(row) {
17
+ const occurredAt = row.occurred_at;
12
18
  return {
13
19
  id: row.id,
14
20
  kind: row.kind,
15
21
  schemaVersion: row.schema_version,
16
- projectId: row.project_id ?? null,
22
+ projectId: row.project_id,
17
23
  sessionId: row.session_id ?? null,
18
24
  planAnchor: row.plan_anchor ?? null,
19
25
  naturalKey: row.natural_key,
20
26
  idempotencyKey: row.idempotency_key,
21
27
  payloadJson: row.payload_json,
22
- syncState: row.sync_state,
23
- attempts: row.attempts,
24
- nextAttemptAt: row.next_attempt_at ?? null,
25
- lastError: row.last_error ?? null,
26
- createdAt: row.created_at,
27
- repositoryRoot: row.repository_root ?? null,
28
- doorUrl: row.door_url ?? null,
29
- placement: row.placement ?? null,
30
- credential: row.credential ?? null,
28
+ writtenByKind: row.written_by_kind,
29
+ occurredAt,
30
+ createdAt: Date.parse(occurredAt),
31
31
  };
32
32
  }
33
33
  export function canonicalJson(value) {
@@ -50,6 +50,13 @@ export function syncStateFor(kind, payload, placement) {
50
50
  return "local";
51
51
  return "pending";
52
52
  }
53
+ export function placementStateName(id) {
54
+ return `placement:${id}`;
55
+ }
56
+ export function storedPlacementOf(db, id) {
57
+ return db.prepare("SELECT 1 FROM sync_state WHERE name = ?").get(placementStateName(id)) !== undefined ? "device" : "server";
58
+ }
59
+ export const REFUSED_ERROR_PREFIX = "refused: ";
53
60
  export class AgentOwnedError extends Error {
54
61
  ownerAgentDefinitionId;
55
62
  naturalKey;
@@ -61,7 +68,7 @@ export class AgentOwnedError extends Error {
61
68
  }
62
69
  }
63
70
  export function findRecordByKey(db, idempotencyKey) {
64
- const row = db.prepare("SELECT * FROM records WHERE idempotency_key = ?").get(idempotencyKey);
71
+ const row = db.prepare("SELECT * FROM harness_records WHERE idempotency_key = ?").get(idempotencyKey);
65
72
  return row ? recordOf(row) : null;
66
73
  }
67
74
  function checkAgentOwned(stored, write, env) {
@@ -77,190 +84,134 @@ function checkAgentOwned(stored, write, env) {
77
84
  return;
78
85
  throw new AgentOwnedError(write.naturalKey, owner);
79
86
  }
80
- function upsert(db, write, placement, syncState) {
81
- db.prepare(`INSERT INTO records (id, kind, schema_version, project_id, session_id, plan_anchor, natural_key, idempotency_key,
82
- payload_json, sync_state, attempts, next_attempt_at, last_error, created_at, repository_root, door_url,
83
- placement, credential)
84
- VALUES (:id, :kind, :schema_version, :project_id, :session_id, :plan_anchor, :natural_key, :idempotency_key,
85
- :payload_json, :sync_state, 0, NULL, NULL, :created_at, :repository_root, NULL, :placement, :credential)
86
- ON CONFLICT(idempotency_key) DO UPDATE SET
87
- kind = excluded.kind, schema_version = excluded.schema_version, project_id = excluded.project_id,
88
- session_id = excluded.session_id, plan_anchor = excluded.plan_anchor, natural_key = excluded.natural_key,
89
- payload_json = excluded.payload_json, sync_state = excluded.sync_state, attempts = 0,
90
- next_attempt_at = NULL, last_error = NULL, created_at = excluded.created_at,
91
- repository_root = excluded.repository_root, door_url = excluded.door_url,
92
- placement = records.placement, credential = excluded.credential`).run({
87
+ function writtenByKindOf(write) {
88
+ return write.credential === "device" ? "agent" : "person";
89
+ }
90
+ function upsert(db, write) {
91
+ const row = db
92
+ .prepare(`INSERT INTO harness_records (id, project_id, kind, schema_version, natural_key, idempotency_key, session_id,
93
+ plan_anchor, payload_json, written_by_kind, occurred_at, created_at, updated_at)
94
+ VALUES (:id, :project_id, :kind, :schema_version, :natural_key, :idempotency_key, :session_id,
95
+ :plan_anchor, :payload_json, :written_by_kind, :occurred_at, datetime('now'), datetime('now'))
96
+ ON CONFLICT (project_id, idempotency_key) DO UPDATE SET
97
+ kind = excluded.kind, schema_version = excluded.schema_version, natural_key = excluded.natural_key,
98
+ session_id = excluded.session_id, plan_anchor = excluded.plan_anchor, payload_json = excluded.payload_json,
99
+ written_by_kind = excluded.written_by_kind, occurred_at = excluded.occurred_at, updated_at = datetime('now')
100
+ RETURNING id`)
101
+ .get({
93
102
  id: randomUUID(),
103
+ project_id: write.projectId ?? unboundProjectId(write.repositoryRoot ?? ""),
94
104
  kind: write.kind,
95
105
  schema_version: write.schemaVersion,
96
- project_id: write.projectId,
97
- session_id: write.sessionId,
98
- plan_anchor: write.planAnchor,
99
106
  natural_key: write.naturalKey,
100
107
  idempotency_key: write.idempotencyKey,
108
+ session_id: write.sessionId,
109
+ plan_anchor: write.planAnchor,
101
110
  payload_json: JSON.stringify(write.payload),
102
- sync_state: syncState,
103
- created_at: write.createdAt,
104
- repository_root: write.repositoryRoot,
105
- placement,
106
- credential: write.credential,
111
+ written_by_kind: writtenByKindOf(write),
112
+ occurred_at: new Date(write.createdAt).toISOString(),
107
113
  });
114
+ return row.id;
108
115
  }
109
116
  export function writeRecords(store, writes, env = process.env) {
110
117
  return withTransaction(store, (db) => writes.map((write) => {
111
118
  const stored = findRecordByKey(db, write.idempotencyKey);
112
119
  checkAgentOwned(stored, write, env);
113
- const placement = stored ? stored.placement : write.placement;
120
+ const id = upsert(db, write);
121
+ const placement = stored ? storedPlacementOf(db, id) : write.placement;
122
+ if (!stored && placement === "device")
123
+ db.prepare("INSERT OR REPLACE INTO sync_state (name, value) VALUES (?, 'device')").run(placementStateName(id));
114
124
  const syncState = syncStateFor(write.kind, write.payload, placement);
115
- upsert(db, write, placement, syncState);
125
+ if (syncState === "local")
126
+ db.prepare("DELETE FROM sync_outbox WHERE table_name = 'harness_records' AND row_key = ?").run(id);
116
127
  return { idempotencyKey: write.idempotencyKey, syncState };
117
128
  }));
118
129
  }
119
130
  export function releaseUnboundRows(store, root) {
120
131
  store.db
121
- .prepare(`UPDATE records SET next_attempt_at = NULL
122
- WHERE project_id IS NULL AND repository_root = ? AND sync_state = 'pending' AND next_attempt_at IS NOT NULL`)
123
- .run(root);
124
- }
125
- export function selectDueRecords(store, now, canSendDevice) {
126
- const rows = store.db
127
- .prepare(`SELECT * FROM records
128
- WHERE sync_state = 'pending' AND (next_attempt_at IS NULL OR next_attempt_at <= ?)
129
- AND (credential IS NULL OR ?)
130
- ORDER BY created_at ASC, rowid ASC`)
131
- .all(now, canSendDevice ? 1 : 0);
132
- return rows.map(recordOf);
132
+ .prepare(`UPDATE sync_outbox SET next_attempt_at = NULL
133
+ WHERE table_name = 'harness_records' AND next_attempt_at IS NOT NULL
134
+ AND row_key IN (SELECT id FROM harness_records WHERE project_id = ?)`)
135
+ .run(unboundProjectId(root));
133
136
  }
134
- export function hasDueRecord(db, now, canSendDevice) {
135
- return (db
136
- .prepare(`SELECT 1 FROM records
137
- WHERE sync_state = 'pending' AND (next_attempt_at IS NULL OR next_attempt_at <= ?)
138
- AND (credential IS NULL OR ?) LIMIT 1`)
139
- .get(now, canSendDevice ? 1 : 0) !== undefined);
140
- }
141
- const CAS = `id = :id AND sync_state = 'pending' AND payload_json = :payload_json AND session_id IS :session_id
142
- AND plan_anchor IS :plan_anchor AND created_at = :created_at`;
143
- function casParameters(row) {
144
- return {
145
- id: row.id,
146
- payload_json: row.payloadJson,
147
- session_id: row.sessionId,
148
- plan_anchor: row.planAnchor,
149
- created_at: row.createdAt,
150
- };
151
- }
152
- export function markSynced(store, row) {
153
- const result = store.db
154
- .prepare(`UPDATE records SET sync_state = 'synced', last_error = NULL, next_attempt_at = NULL WHERE ${CAS}`)
155
- .run(casParameters(row));
156
- return Number(result.changes) > 0;
157
- }
158
- export function markRefused(store, row, error) {
159
- const result = store.db
160
- .prepare(`UPDATE records SET sync_state = 'refused', last_error = :last_error, next_attempt_at = NULL WHERE ${CAS}`)
161
- .run({ ...casParameters(row), last_error: error });
162
- return Number(result.changes) > 0;
163
- }
164
- export function markBackoff(store, row, error, nextAttemptAt) {
165
- const result = store.db
166
- .prepare(`UPDATE records SET attempts = attempts + 1, next_attempt_at = :next_attempt_at, last_error = :last_error WHERE ${CAS}`)
167
- .run({ ...casParameters(row), last_error: error, next_attempt_at: nextAttemptAt });
168
- return Number(result.changes) > 0;
169
- }
170
- export function markDeferred(store, row, nextAttemptAt) {
171
- const result = store.db
172
- .prepare(`UPDATE records SET next_attempt_at = :next_attempt_at WHERE ${CAS}`)
173
- .run({ ...casParameters(row), next_attempt_at: nextAttemptAt });
174
- return Number(result.changes) > 0;
137
+ function dropUnpushed(db, id) {
138
+ db.prepare("DELETE FROM harness_records WHERE id = ?").run(id);
139
+ db.prepare("DELETE FROM sync_outbox WHERE table_name = 'harness_records' AND row_key = ?").run(id);
175
140
  }
176
141
  export function rekeyRecord(store, row, projectId) {
177
142
  const key = `${projectId}:${row.kind}:${row.naturalKey}`;
178
143
  return withTransaction(store, (db) => {
179
- const current = db.prepare("SELECT * FROM records WHERE id = ?").get(row.id);
144
+ const current = db.prepare("SELECT * FROM harness_records WHERE id = ?").get(row.id);
180
145
  if (!current)
181
146
  return null;
182
- const existing = findRecordByKey(db, key);
183
- if (existing && existing.id !== row.id) {
184
- if (existing.createdAt >= row.createdAt) {
185
- db.prepare("DELETE FROM records WHERE id = ?").run(row.id);
147
+ const existing = db.prepare("SELECT * FROM harness_records WHERE project_id = ? AND idempotency_key = ?").get(projectId, key);
148
+ if (existing) {
149
+ const stored = recordOf(existing);
150
+ dropUnpushed(db, row.id);
151
+ if (stored.createdAt >= row.createdAt)
186
152
  return null;
187
- }
188
- db.prepare("DELETE FROM records WHERE id = ?").run(existing.id);
153
+ db.prepare(`UPDATE harness_records SET kind = ?, schema_version = ?, natural_key = ?, session_id = ?, plan_anchor = ?,
154
+ payload_json = ?, written_by_kind = ?, occurred_at = ?, updated_at = datetime('now') WHERE id = ?`).run(row.kind, row.schemaVersion, row.naturalKey, row.sessionId, row.planAnchor, row.payloadJson, row.writtenByKind, row.occurredAt, stored.id);
155
+ return recordOf(db.prepare("SELECT * FROM harness_records WHERE id = ?").get(stored.id));
189
156
  }
190
- db.prepare("UPDATE records SET project_id = ?, idempotency_key = ? WHERE id = ?").run(projectId, key, row.id);
191
- return recordOf(db.prepare("SELECT * FROM records WHERE id = ?").get(row.id));
157
+ db.prepare("UPDATE harness_records SET project_id = ?, idempotency_key = ?, updated_at = datetime('now') WHERE id = ?").run(projectId, key, row.id);
158
+ return recordOf(db.prepare("SELECT * FROM harness_records WHERE id = ?").get(row.id));
192
159
  });
193
160
  }
194
- export function pruneSynced(store, now) {
195
- store.db
196
- .prepare(`DELETE FROM records WHERE sync_state = 'synced' AND created_at < ?
197
- AND NOT (kind = 'obligation' AND json_extract(payload_json, '$.state') = 'open')`)
198
- .run(now - SYNCED_RETENTION_DAYS * 24 * 60 * 60 * 1000);
199
- }
161
+ const VIEW_SELECT = `SELECT r.*, o.row_key AS outbox_key, o.attempts AS outbox_attempts, o.next_attempt_at AS outbox_next_attempt_at,
162
+ o.last_error AS outbox_last_error
163
+ FROM harness_records r LEFT JOIN sync_outbox o ON o.table_name = 'harness_records' AND o.row_key = r.id`;
200
164
  function viewOf(row) {
165
+ const stored = recordOf(row);
201
166
  let payload;
202
167
  try {
203
- payload = JSON.parse(row.payloadJson);
168
+ payload = JSON.parse(stored.payloadJson);
204
169
  }
205
170
  catch {
206
- payload = row.payloadJson;
171
+ payload = stored.payloadJson;
207
172
  }
173
+ const lastError = row.outbox_last_error ?? null;
174
+ const syncState = row.outbox_key !== null && row.outbox_key !== undefined
175
+ ? lastError?.startsWith(REFUSED_ERROR_PREFIX)
176
+ ? "refused"
177
+ : "pending"
178
+ : row.sync_seq !== null
179
+ ? "synced"
180
+ : "local";
181
+ const root = rootOfUnbound(stored.projectId);
208
182
  return {
209
- id: row.id,
210
- kind: row.kind,
211
- projectId: row.projectId,
212
- sessionId: row.sessionId,
213
- planAnchor: row.planAnchor,
214
- naturalKey: row.naturalKey,
215
- idempotencyKey: row.idempotencyKey,
183
+ id: stored.id,
184
+ kind: stored.kind,
185
+ projectId: root === null ? stored.projectId : null,
186
+ sessionId: stored.sessionId,
187
+ planAnchor: stored.planAnchor,
188
+ naturalKey: stored.naturalKey,
189
+ idempotencyKey: stored.idempotencyKey,
216
190
  payload,
217
- syncState: row.syncState,
218
- attempts: row.attempts,
219
- nextAttemptAt: row.nextAttemptAt,
220
- lastError: row.lastError,
221
- createdAt: row.createdAt,
222
- repositoryRoot: row.repositoryRoot,
223
- placement: row.placement,
224
- credential: row.credential,
191
+ syncState,
192
+ attempts: Number(row.outbox_attempts ?? 0),
193
+ nextAttemptAt: row.outbox_next_attempt_at ?? null,
194
+ lastError,
195
+ createdAt: stored.createdAt,
196
+ repositoryRoot: root,
197
+ credential: stored.writtenByKind === "agent" ? "device" : null,
225
198
  };
226
199
  }
227
- export async function countRecords(store, projectId) {
228
- const rows = (projectId
229
- ? store.db.prepare("SELECT * FROM records WHERE sync_state <> 'synced' AND project_id = ?").all(projectId)
230
- : store.db.prepare("SELECT * FROM records WHERE sync_state <> 'synced'").all());
231
- const records = rows.map(recordOf);
232
- const doorOff = new Map();
233
- for (const row of records) {
234
- if (row.syncState !== "pending" || row.projectId === null || row.doorUrl !== null || row.repositoryRoot === null)
235
- continue;
236
- if (!doorOff.has(row.repositoryRoot))
237
- doorOff.set(row.repositoryRoot, (await readMountDoor(row.repositoryRoot)) === null);
238
- }
239
- const isDoorOff = (row) => row.doorUrl === null && row.repositoryRoot !== null && doorOff.get(row.repositoryRoot) === true;
240
- let waitingCount = 0;
241
- let refusedWithProject = 0;
242
- let storeCount = 0;
243
- for (const row of records) {
244
- if (row.syncState === "refused" && row.projectId !== null)
245
- refusedWithProject += 1;
246
- if (row.syncState !== "pending")
247
- continue;
248
- if (row.projectId === null || isDoorOff(row))
249
- storeCount += 1;
250
- else
251
- waitingCount += 1;
252
- }
253
- const outboxCount = waitingCount + refusedWithProject;
254
- const newest = records
255
- .filter((row) => (row.syncState === "pending" || row.syncState === "refused") && row.lastError !== null)
256
- .sort((a, b) => b.createdAt - a.createdAt)[0];
200
+ export function countRecords(store, projectId) {
201
+ const where = `WHERE (o.row_key IS NOT NULL OR r.sync_seq IS NULL)${projectId ? " AND r.project_id = ?" : ""}`;
202
+ const rows = (projectId ? store.db.prepare(`${VIEW_SELECT} ${where}`).all(projectId) : store.db.prepare(`${VIEW_SELECT} ${where}`).all());
203
+ const views = rows.map(viewOf);
204
+ const isBound = (view) => view.repositoryRoot === null;
205
+ const waitingCount = views.filter((view) => view.syncState === "pending" && isBound(view)).length;
206
+ const refusedWithProject = views.filter((view) => view.syncState === "refused" && isBound(view)).length;
207
+ const newest = views.filter((view) => view.lastError !== null).sort((a, b) => b.createdAt - a.createdAt)[0];
257
208
  return {
258
- outboxCount,
259
- storeCount,
209
+ outboxCount: waitingCount + refusedWithProject,
210
+ storeCount: views.filter((view) => view.syncState === "pending" && !isBound(view)).length,
260
211
  lastError: newest?.lastError ?? null,
261
- pendingCount: records.filter((row) => row.syncState === "pending").length,
262
- refusedCount: records.filter((row) => row.syncState === "refused").length,
263
- localCount: records.filter((row) => row.syncState === "local").length,
212
+ pendingCount: views.filter((view) => view.syncState === "pending").length,
213
+ refusedCount: views.filter((view) => view.syncState === "refused").length,
214
+ localCount: views.filter((view) => view.syncState === "local").length,
264
215
  waitingCount,
265
216
  };
266
217
  }
@@ -268,20 +219,27 @@ export function listRecords(store, filter = {}) {
268
219
  const clauses = [];
269
220
  const parameters = [];
270
221
  if (filter.projectId) {
271
- clauses.push("project_id = ?");
222
+ clauses.push("r.project_id = ?");
272
223
  parameters.push(filter.projectId);
273
224
  }
274
225
  if (filter.kind) {
275
- clauses.push("kind = ?");
226
+ clauses.push("r.kind = ?");
276
227
  parameters.push(filter.kind);
277
228
  }
278
229
  if (filter.planAnchor) {
279
- clauses.push("plan_anchor = ?");
230
+ clauses.push("r.plan_anchor = ?");
280
231
  parameters.push(filter.planAnchor);
281
232
  }
282
233
  const where = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : "";
283
234
  const rows = store.db
284
- .prepare(`SELECT * FROM records ${where} ORDER BY created_at DESC, rowid DESC LIMIT ${RECORDS_LIST_LIMIT}`)
235
+ .prepare(`${VIEW_SELECT} ${where} ORDER BY r.occurred_at DESC, r.rowid DESC LIMIT ${RECORDS_LIST_LIMIT}`)
285
236
  .all(...parameters);
286
- return rows.map(recordOf).map(viewOf);
237
+ return rows.map(viewOf);
238
+ }
239
+ export function selectUnboundRecords(db) {
240
+ const rows = db
241
+ .prepare(`SELECT r.* FROM harness_records r JOIN sync_outbox o ON o.table_name = 'harness_records' AND o.row_key = r.id
242
+ WHERE r.project_id LIKE '${UNBOUND_PROJECT_PREFIX}%' ORDER BY r.occurred_at, r.rowid`)
243
+ .all();
244
+ return rows.map(recordOf);
287
245
  }
@@ -1,7 +1,7 @@
1
1
  import { randomUUID } from "node:crypto";
2
- import { RECORD_SCHEMA_VERSION } from "./door.js";
2
+ import { RECORD_SCHEMA_VERSION } from "./kinds.js";
3
3
  import { idempotencyKeyOf } from "./naturalKey.js";
4
- import { writeRecords } from "./recordsTable.js";
4
+ import { UNBOUND_PROJECT_PREFIX, writeRecords } from "./recordsTable.js";
5
5
  export const REFUSAL_GATE_IDS = [
6
6
  "records-validation",
7
7
  "records-unknown-kind",
@@ -69,19 +69,19 @@ export function countRefusals(store, filter) {
69
69
  const ids = REFUSAL_GATE_IDS.map(() => "?").join(", ");
70
70
  const clauses = [
71
71
  "kind = 'gate_event'",
72
- "project_id IS ?",
72
+ filter.projectId === null ? `project_id LIKE '${UNBOUND_PROJECT_PREFIX}%'` : "project_id = ?",
73
73
  `json_extract(payload_json, '$.gate') IN (${ids})`,
74
74
  "json_extract(payload_json, '$.verdict') = ?",
75
75
  ];
76
- const parameters = [filter.projectId, ...REFUSAL_GATE_IDS, filter.verdict ?? "block"];
76
+ const parameters = [...(filter.projectId === null ? [] : [filter.projectId]), ...REFUSAL_GATE_IDS, filter.verdict ?? "block"];
77
77
  if (filter.sessionId !== undefined) {
78
78
  clauses.push("session_id = ?");
79
79
  parameters.push(filter.sessionId);
80
80
  }
81
81
  if (filter.since !== undefined) {
82
- clauses.push("created_at >= ?");
83
- parameters.push(filter.since);
82
+ clauses.push("occurred_at >= ?");
83
+ parameters.push(new Date(filter.since).toISOString());
84
84
  }
85
- const row = store.db.prepare(`SELECT count(*) AS n FROM records WHERE ${clauses.join(" AND ")}`).get(...parameters);
85
+ const row = store.db.prepare(`SELECT count(*) AS n FROM harness_records WHERE ${clauses.join(" AND ")}`).get(...parameters);
86
86
  return Number(row.n);
87
87
  }
@@ -3,7 +3,7 @@ import path from "node:path";
3
3
  import { ensureAccessToken } from "../auth_grant.js";
4
4
  import { briefCachePath } from "../local_store/paths.js";
5
5
  import { assertCredentialOrigin } from "../nustack_client.js";
6
- import { resolveDoor } from "../records/door.js";
6
+ import { resolveDoor } from "../sync/endpoint.js";
7
7
  export const BRIEF_FETCH_TIMEOUT_MS = 3_000;
8
8
  export const BRIEF_CONTEXT_CAP = 4_000;
9
9
  export const BRIEF_CONTEXT_OBLIGATIONS = 20;
@@ -2,8 +2,8 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, unlinkSyn
2
2
  import path from "node:path";
3
3
  import { ensureAccessToken } from "../auth_grant.js";
4
4
  import { sessionEventsDir } from "../local_store/paths.js";
5
- import { labelRowWire, listPendingLabelRows, markLabelRowsRefused, markLabelRowsSynced } from "../local_store/routing.js";
6
- import { RECORDS_BATCH_CAP, resolveDoor } from "../records/door.js";
5
+ import { RECORDS_BATCH_CAP } from "../records/kinds.js";
6
+ import { resolveDoor } from "../sync/endpoint.js";
7
7
  import { makeSessionsClient, postSessionEvents } from "./client.js";
8
8
  export const SESSION_SPOOL_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
9
9
  const SESSION_DRAIN_PASSES = 3;
@@ -120,52 +120,9 @@ export async function drainSessionEvents(globals, result, bearers = new Map()) {
120
120
  }
121
121
  }
122
122
  }
123
- function spoolOf(sessionId, spools) {
123
+ export function spoolOf(sessionId, spools) {
124
124
  return spools.find((spool) => spool.rowId === sessionId) ?? spools.find((spool) => spool.runtimeSessionId === sessionId) ?? null;
125
125
  }
126
- export async function drainLabelRows(store, globals, result, bearers = new Map(), now = Date.now()) {
127
- const rows = listPendingLabelRows(store, RECORDS_BATCH_CAP);
128
- if (rows.length === 0)
129
- return;
130
- const spools = listSpools();
131
- const groups = new Map();
132
- const stale = [];
133
- for (const record of rows) {
134
- const spool = spoolOf(record.session_id, spools);
135
- if (!spool) {
136
- if (now - record.created_at > SESSION_SPOOL_RETENTION_MS)
137
- stale.push(record.id);
138
- continue;
139
- }
140
- const key = `${spool.root}\u0000${spool.projectId}`;
141
- const group = groups.get(key) ?? { spool, rows: [] };
142
- group.rows.push({ record, wire: { ...labelRowWire(record), sessionId: spool.rowId } });
143
- groups.set(key, group);
144
- }
145
- if (stale.length > 0)
146
- result.labelRowsRefused += markLabelRowsRefused(store, stale);
147
- for (const { spool, rows: batch } of groups.values()) {
148
- const door = await doorFor(spool.root, globals, bearers);
149
- if (!door)
150
- continue;
151
- const request = { events: [], labelRows: batch.map((row) => row.wire) };
152
- const outcome = await postSessionEvents(door.doorUrl, door.bearer, spool.projectId, request);
153
- if (outcome.ok) {
154
- result.labelRowsSynced += markLabelRowsSynced(store, batch.map((row) => row.record.id));
155
- continue;
156
- }
157
- result.lastError = outcome.message;
158
- if (!isRefusal(outcome))
159
- continue;
160
- for (const row of batch) {
161
- const alone = await postSessionEvents(door.doorUrl, door.bearer, spool.projectId, { events: [], labelRows: [row.wire] });
162
- if (alone.ok)
163
- result.labelRowsSynced += markLabelRowsSynced(store, [row.record.id]);
164
- else if (isRefusal(alone))
165
- result.labelRowsRefused += markLabelRowsRefused(store, [row.record.id]);
166
- }
167
- }
168
- }
169
126
  export function pruneSpools(now = Date.now()) {
170
127
  let pruned = 0;
171
128
  for (const spool of listSpools()) {
@@ -182,8 +139,8 @@ export function pruneSpools(now = Date.now()) {
182
139
  }
183
140
  return pruned;
184
141
  }
185
- export async function drainSessionLegs(store, globals) {
186
- const result = { eventsSent: 0, eventsDropped: 0, closed: 0, labelRowsSynced: 0, labelRowsRefused: 0, lastError: null };
142
+ export async function drainSessionLegs(globals) {
143
+ const result = { eventsSent: 0, eventsDropped: 0, closed: 0, lastError: null };
187
144
  const bearers = new Map();
188
145
  try {
189
146
  for (let pass = 0; pass < SESSION_DRAIN_PASSES; pass += 1) {
@@ -193,7 +150,6 @@ export async function drainSessionLegs(store, globals) {
193
150
  if (isIdle || result.eventsSent + result.eventsDropped + result.closed === before)
194
151
  break;
195
152
  }
196
- await drainLabelRows(store, globals, result, bearers);
197
153
  pruneSpools();
198
154
  }
199
155
  catch (error) {
@@ -0,0 +1,136 @@
1
+ import { existsSync, readFileSync, rmSync } from "node:fs";
2
+ import { chmod } from "node:fs/promises";
3
+ import os from "node:os";
4
+ import path from "node:path";
5
+ import { mkdirSecure } from "../secure_file.js";
6
+ import { nustackHomeDir } from "../local_store/paths.js";
7
+ import { StoreTooNewError, versionVerdict, wipeStaleStore } from "./wipe.js";
8
+ export function machineStorePath(home = os.homedir()) {
9
+ return path.join(home, ".nustack", "nustack.db");
10
+ }
11
+ export function databaseFiles(dbFile) {
12
+ return [dbFile, `${dbFile}-wal`, `${dbFile}-shm`];
13
+ }
14
+ export function storeStampOf(version) {
15
+ const match = /^v?(\d+)\.(\d+)/.exec(version.trim());
16
+ if (!match)
17
+ throw new Error(`not a release number: ${version}`);
18
+ return Number(match[1]) * 1000 + Number(match[2]);
19
+ }
20
+ let cliVersionCache;
21
+ export function cliVersion() {
22
+ cliVersionCache ??= JSON.parse(readFileSync(new URL("../../../package.json", import.meta.url), "utf8")).version;
23
+ return cliVersionCache;
24
+ }
25
+ let schemaCache;
26
+ function machineSchemaSql() {
27
+ schemaCache ??= readFileSync(new URL("../../../dist/machine.sql", import.meta.url), "utf8");
28
+ return schemaCache;
29
+ }
30
+ let sqliteModulePromise;
31
+ export async function withSuppressedSqliteWarning(fn) {
32
+ const original = process.emitWarning;
33
+ const filtered = (warning, ...args) => {
34
+ const name = warning instanceof Error
35
+ ? warning.name
36
+ : typeof args[0] === "string"
37
+ ? args[0]
38
+ : args[0]?.type;
39
+ const message = warning instanceof Error ? warning.message : String(warning);
40
+ if (name === "ExperimentalWarning" && /sqlite/i.test(message))
41
+ return;
42
+ original(warning, ...args);
43
+ };
44
+ process.emitWarning = filtered;
45
+ try {
46
+ return await fn();
47
+ }
48
+ finally {
49
+ process.emitWarning = original;
50
+ }
51
+ }
52
+ export async function openSqliteFile(dbFile, options = {}) {
53
+ sqliteModulePromise ??= withSuppressedSqliteWarning(() => import("node:sqlite"));
54
+ const { DatabaseSync } = await sqliteModulePromise;
55
+ return new DatabaseSync(dbFile, options.readOnly ? { readOnly: true } : {});
56
+ }
57
+ function userVersionOf(db) {
58
+ const row = db.prepare("PRAGMA user_version").get();
59
+ return Number(row?.user_version ?? 0);
60
+ }
61
+ function retireReplacedStores() {
62
+ for (const file of databaseFiles(path.join(nustackHomeDir(), "state.db"))) {
63
+ try {
64
+ rmSync(file, { force: true });
65
+ }
66
+ catch {
67
+ }
68
+ }
69
+ }
70
+ function settleSchema(db, stamp) {
71
+ if (userVersionOf(db) === stamp)
72
+ return true;
73
+ db.exec("BEGIN IMMEDIATE");
74
+ try {
75
+ const found = userVersionOf(db);
76
+ const tableCount = Number(db.prepare("SELECT count(*) AS n FROM sqlite_master").get().n);
77
+ if (found === 0 && tableCount === 0) {
78
+ db.exec(machineSchemaSql());
79
+ db.exec(`PRAGMA user_version = ${stamp}`);
80
+ db.exec("COMMIT");
81
+ return true;
82
+ }
83
+ db.exec("ROLLBACK");
84
+ const verdict = versionVerdict(found, stamp);
85
+ if (verdict === "same")
86
+ return true;
87
+ if (verdict === "newer")
88
+ throw new StoreTooNewError(found, stamp);
89
+ return false;
90
+ }
91
+ catch (error) {
92
+ try {
93
+ db.exec("ROLLBACK");
94
+ }
95
+ catch {
96
+ }
97
+ throw error;
98
+ }
99
+ }
100
+ export async function openMachineDatabase(dbFile = machineStorePath()) {
101
+ retireReplacedStores();
102
+ await mkdirSecure(path.dirname(dbFile));
103
+ const stamp = storeStampOf(cliVersion());
104
+ for (let attempt = 0; attempt < 2; attempt += 1) {
105
+ const db = await withSuppressedSqliteWarning(() => openSqliteFile(dbFile));
106
+ try {
107
+ db.exec("PRAGMA busy_timeout = 5000");
108
+ db.exec("PRAGMA journal_mode = WAL");
109
+ db.exec("PRAGMA foreign_keys = OFF");
110
+ if (settleSchema(db, stamp)) {
111
+ await chmod(dbFile, 0o600).catch(() => { });
112
+ return db;
113
+ }
114
+ }
115
+ catch (error) {
116
+ db.close();
117
+ throw error;
118
+ }
119
+ await wipeStaleStore(db, dbFile);
120
+ }
121
+ throw new Error(`the machine store at ${dbFile} could not be recreated`);
122
+ }
123
+ export async function openMachineDatabaseReadOnly(dbFile = machineStorePath()) {
124
+ if (!existsSync(dbFile))
125
+ return null;
126
+ const db = await withSuppressedSqliteWarning(() => openSqliteFile(dbFile, { readOnly: true }));
127
+ try {
128
+ db.exec("PRAGMA busy_timeout = 2000");
129
+ if (userVersionOf(db) === storeStampOf(cliVersion()))
130
+ return db;
131
+ }
132
+ catch {
133
+ }
134
+ db.close();
135
+ return null;
136
+ }