mdkg 0.1.8 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/README.md +10 -4
- package/dist/cli.js +4 -2
- package/dist/core/project_db_events.js +599 -0
- package/dist/core/project_db_materializer.js +462 -0
- package/dist/core/project_db_migrations.js +97 -1
- package/dist/init/AGENT_START.md +7 -4
- package/dist/init/CLI_COMMAND_MATRIX.md +5 -1
- package/dist/init/README.md +7 -4
- package/dist/init/init-manifest.json +4 -4
- package/package.json +4 -2
|
@@ -0,0 +1,462 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.PROJECT_DB_MATERIALIZER_SCHEMA_VERSION = exports.PROJECT_DB_MATERIALIZER_KIND = exports.PROJECT_DB_MATERIALIZER_QUEUE = void 0;
|
|
7
|
+
exports.enqueueProjectDbMaterialization = enqueueProjectDbMaterialization;
|
|
8
|
+
exports.runNextProjectDbMaterializer = runNextProjectDbMaterializer;
|
|
9
|
+
exports.readProjectDbMaterializerStats = readProjectDbMaterializerStats;
|
|
10
|
+
const crypto_1 = __importDefault(require("crypto"));
|
|
11
|
+
const fs_1 = __importDefault(require("fs"));
|
|
12
|
+
const path_1 = __importDefault(require("path"));
|
|
13
|
+
const config_1 = require("./config");
|
|
14
|
+
const project_db_queue_1 = require("./project_db_queue");
|
|
15
|
+
const project_db_events_1 = require("./project_db_events");
|
|
16
|
+
const project_db_snapshot_1 = require("./project_db_snapshot");
|
|
17
|
+
exports.PROJECT_DB_MATERIALIZER_QUEUE = "project-db.materialize";
|
|
18
|
+
exports.PROJECT_DB_MATERIALIZER_KIND = "project-db.materialize";
|
|
19
|
+
exports.PROJECT_DB_MATERIALIZER_SCHEMA_VERSION = 1;
|
|
20
|
+
function loadDatabaseCtor() {
|
|
21
|
+
try {
|
|
22
|
+
const loaded = require("node:sqlite");
|
|
23
|
+
if (!loaded.DatabaseSync) {
|
|
24
|
+
throw new Error("node:sqlite DatabaseSync is unavailable");
|
|
25
|
+
}
|
|
26
|
+
return loaded.DatabaseSync;
|
|
27
|
+
}
|
|
28
|
+
catch (err) {
|
|
29
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
30
|
+
throw new Error(`node:sqlite is required for mdkg project DB materializers: ${message}`);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
function nowMs(input) {
|
|
34
|
+
if (input !== undefined) {
|
|
35
|
+
assertInteger(input, "now_ms");
|
|
36
|
+
return input;
|
|
37
|
+
}
|
|
38
|
+
return Date.now();
|
|
39
|
+
}
|
|
40
|
+
function assertNonEmpty(value, field) {
|
|
41
|
+
if (typeof value !== "string" || value.trim() === "") {
|
|
42
|
+
throw new Error(`${field} must be a non-empty string`);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
function assertInteger(value, field) {
|
|
46
|
+
if (!Number.isInteger(value)) {
|
|
47
|
+
throw new Error(`${field} must be an integer`);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
function assertPositiveInteger(value, field) {
|
|
51
|
+
assertInteger(value, field);
|
|
52
|
+
if (value <= 0) {
|
|
53
|
+
throw new Error(`${field} must be greater than 0`);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
function sha256File(filePath) {
|
|
57
|
+
return `sha256:${crypto_1.default.createHash("sha256").update(fs_1.default.readFileSync(filePath)).digest("hex")}`;
|
|
58
|
+
}
|
|
59
|
+
function safeSegment(value) {
|
|
60
|
+
return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "materializer";
|
|
61
|
+
}
|
|
62
|
+
function isObject(value) {
|
|
63
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
64
|
+
}
|
|
65
|
+
function stringField(value, field) {
|
|
66
|
+
const raw = value[field];
|
|
67
|
+
if (typeof raw !== "string" || raw.trim() === "") {
|
|
68
|
+
throw new Error(`materializer payload ${field} must be a non-empty string`);
|
|
69
|
+
}
|
|
70
|
+
return raw;
|
|
71
|
+
}
|
|
72
|
+
function parsePayload(payloadJson) {
|
|
73
|
+
let parsed;
|
|
74
|
+
try {
|
|
75
|
+
parsed = JSON.parse(payloadJson);
|
|
76
|
+
}
|
|
77
|
+
catch (err) {
|
|
78
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
79
|
+
throw new Error(`materializer payload must be valid JSON: ${message}`);
|
|
80
|
+
}
|
|
81
|
+
if (!isObject(parsed)) {
|
|
82
|
+
throw new Error("materializer payload must be an object");
|
|
83
|
+
}
|
|
84
|
+
if (parsed.kind !== exports.PROJECT_DB_MATERIALIZER_KIND) {
|
|
85
|
+
throw new Error(`materializer payload kind must be ${exports.PROJECT_DB_MATERIALIZER_KIND}`);
|
|
86
|
+
}
|
|
87
|
+
if (parsed.schema_version !== exports.PROJECT_DB_MATERIALIZER_SCHEMA_VERSION) {
|
|
88
|
+
throw new Error(`materializer payload schema_version must be ${exports.PROJECT_DB_MATERIALIZER_SCHEMA_VERSION}`);
|
|
89
|
+
}
|
|
90
|
+
const reducerName = stringField(parsed, "reducer_name");
|
|
91
|
+
if (reducerName !== "project_meta.set") {
|
|
92
|
+
throw new Error(`unsupported materializer reducer_name: ${reducerName}`);
|
|
93
|
+
}
|
|
94
|
+
const reducerVersion = stringField(parsed, "reducer_version");
|
|
95
|
+
if (reducerVersion !== "v1") {
|
|
96
|
+
throw new Error(`unsupported materializer reducer_version: ${reducerVersion}`);
|
|
97
|
+
}
|
|
98
|
+
return {
|
|
99
|
+
kind: exports.PROJECT_DB_MATERIALIZER_KIND,
|
|
100
|
+
schema_version: exports.PROJECT_DB_MATERIALIZER_SCHEMA_VERSION,
|
|
101
|
+
project_id: stringField(parsed, "project_id"),
|
|
102
|
+
branch_id: stringField(parsed, "branch_id"),
|
|
103
|
+
event_id: stringField(parsed, "event_id"),
|
|
104
|
+
reducer_name: reducerName,
|
|
105
|
+
reducer_version: reducerVersion,
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
function partialPayloadIds(payloadJson) {
|
|
109
|
+
try {
|
|
110
|
+
const parsed = JSON.parse(payloadJson);
|
|
111
|
+
if (isObject(parsed)) {
|
|
112
|
+
return {
|
|
113
|
+
project_id: typeof parsed.project_id === "string" && parsed.project_id.trim() !== "" ? parsed.project_id : "unknown",
|
|
114
|
+
branch_id: typeof parsed.branch_id === "string" && parsed.branch_id.trim() !== "" ? parsed.branch_id : "unknown",
|
|
115
|
+
event_id: typeof parsed.event_id === "string" && parsed.event_id.trim() !== "" ? parsed.event_id : null,
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
catch {
|
|
120
|
+
// fall through to unknown identifiers for rejected payload receipts
|
|
121
|
+
}
|
|
122
|
+
return { project_id: "unknown", branch_id: "unknown", event_id: null };
|
|
123
|
+
}
|
|
124
|
+
function findRepoRoot(databasePath, explicitRoot) {
|
|
125
|
+
if (explicitRoot) {
|
|
126
|
+
return path_1.default.resolve(explicitRoot);
|
|
127
|
+
}
|
|
128
|
+
let current = path_1.default.dirname(path_1.default.resolve(databasePath));
|
|
129
|
+
while (true) {
|
|
130
|
+
if (fs_1.default.existsSync(path_1.default.join(current, ".mdkg", "config.json"))) {
|
|
131
|
+
return current;
|
|
132
|
+
}
|
|
133
|
+
const parent = path_1.default.dirname(current);
|
|
134
|
+
if (parent === current) {
|
|
135
|
+
throw new Error("repo_root is required when databasePath is not inside an mdkg repo");
|
|
136
|
+
}
|
|
137
|
+
current = parent;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
function withDb(databasePath, fn) {
|
|
141
|
+
const DatabaseSync = loadDatabaseCtor();
|
|
142
|
+
const db = new DatabaseSync(databasePath);
|
|
143
|
+
try {
|
|
144
|
+
db.exec("PRAGMA foreign_keys = ON;");
|
|
145
|
+
return fn(db);
|
|
146
|
+
}
|
|
147
|
+
finally {
|
|
148
|
+
db.close();
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
function readBranchSnapshotHash(databasePath, projectId, branchId) {
|
|
152
|
+
return withDb(databasePath, (db) => {
|
|
153
|
+
const row = db
|
|
154
|
+
.prepare("SELECT current_snapshot_hash FROM project_branch_state WHERE project_id = ? AND branch_id = ?")
|
|
155
|
+
.get(projectId, branchId);
|
|
156
|
+
return row?.current_snapshot_hash === undefined || row.current_snapshot_hash === null
|
|
157
|
+
? null
|
|
158
|
+
: String(row.current_snapshot_hash);
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
function queueName(input) {
|
|
162
|
+
if (input !== undefined) {
|
|
163
|
+
assertNonEmpty(input, "queue_name");
|
|
164
|
+
return input;
|
|
165
|
+
}
|
|
166
|
+
return exports.PROJECT_DB_MATERIALIZER_QUEUE;
|
|
167
|
+
}
|
|
168
|
+
function result(input) {
|
|
169
|
+
return {
|
|
170
|
+
status: input.status,
|
|
171
|
+
queue_name: input.queue_name,
|
|
172
|
+
queue_message: input.queue_message,
|
|
173
|
+
payload: input.payload ?? null,
|
|
174
|
+
reducer: input.reducer ?? null,
|
|
175
|
+
lease: input.lease ?? null,
|
|
176
|
+
receipt: input.receipt ?? null,
|
|
177
|
+
snapshot: input.snapshot ?? null,
|
|
178
|
+
error: input.error ?? null,
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
function failStatus(message, fallback) {
|
|
182
|
+
return message.status === "dead_letter" ? "dead_letter" : fallback;
|
|
183
|
+
}
|
|
184
|
+
function writeMaterializerReceipt(databasePath, input) {
|
|
185
|
+
return (0, project_db_events_1.writeProjectDbReceipt)(databasePath, {
|
|
186
|
+
project_id: input.payload.project_id,
|
|
187
|
+
branch_id: input.payload.branch_id,
|
|
188
|
+
kind: input.kind,
|
|
189
|
+
status: input.status,
|
|
190
|
+
event_id: input.payload.event_id,
|
|
191
|
+
actor: input.actor,
|
|
192
|
+
details: {
|
|
193
|
+
message_id: input.message.message_id,
|
|
194
|
+
queue_name: input.message.queue_name,
|
|
195
|
+
error: input.error,
|
|
196
|
+
},
|
|
197
|
+
now_ms: input.now_ms,
|
|
198
|
+
receipts_path: input.receipts_path,
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
function tryReleaseWriterLease(databasePath, payload, leaseId, leaseOwner, now, receiptsPath) {
|
|
202
|
+
try {
|
|
203
|
+
(0, project_db_events_1.releaseProjectWriterLease)(databasePath, {
|
|
204
|
+
project_id: payload.project_id,
|
|
205
|
+
branch_id: payload.branch_id,
|
|
206
|
+
lease_id: leaseId,
|
|
207
|
+
lease_owner: leaseOwner,
|
|
208
|
+
now_ms: now,
|
|
209
|
+
receipts_path: receiptsPath,
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
catch {
|
|
213
|
+
// The lease may already be committed, conflicted, released, or absent.
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
function enqueueProjectDbMaterialization(databasePath, input) {
|
|
217
|
+
assertNonEmpty(input.message_id, "message_id");
|
|
218
|
+
assertNonEmpty(input.project_id, "project_id");
|
|
219
|
+
assertNonEmpty(input.branch_id, "branch_id");
|
|
220
|
+
assertNonEmpty(input.event_id, "event_id");
|
|
221
|
+
const selectedQueue = queueName(input.queue_name);
|
|
222
|
+
const payload = {
|
|
223
|
+
kind: exports.PROJECT_DB_MATERIALIZER_KIND,
|
|
224
|
+
schema_version: exports.PROJECT_DB_MATERIALIZER_SCHEMA_VERSION,
|
|
225
|
+
project_id: input.project_id,
|
|
226
|
+
branch_id: input.branch_id,
|
|
227
|
+
event_id: input.event_id,
|
|
228
|
+
reducer_name: input.reducer_name,
|
|
229
|
+
reducer_version: input.reducer_version,
|
|
230
|
+
};
|
|
231
|
+
return (0, project_db_queue_1.enqueueProjectQueueMessage)(databasePath, {
|
|
232
|
+
queue_name: selectedQueue,
|
|
233
|
+
message_id: input.message_id,
|
|
234
|
+
dedupe_key: input.dedupe_key === undefined
|
|
235
|
+
? `${input.project_id}:${input.branch_id}:${input.event_id}:${input.reducer_name}:${input.reducer_version}`
|
|
236
|
+
: input.dedupe_key,
|
|
237
|
+
payload,
|
|
238
|
+
available_at_ms: input.available_at_ms,
|
|
239
|
+
max_attempts: input.max_attempts,
|
|
240
|
+
now_ms: input.now_ms,
|
|
241
|
+
});
|
|
242
|
+
}
|
|
243
|
+
function runNextProjectDbMaterializer(databasePath, input) {
|
|
244
|
+
assertNonEmpty(input.lease_owner, "lease_owner");
|
|
245
|
+
assertPositiveInteger(input.lease_ms, "lease_ms");
|
|
246
|
+
const selectedQueue = queueName(input.queue_name);
|
|
247
|
+
const currentNow = nowMs(input.now_ms);
|
|
248
|
+
const retryAfter = input.retry_after_ms ?? 0;
|
|
249
|
+
assertInteger(retryAfter, "retry_after_ms");
|
|
250
|
+
if (retryAfter < 0) {
|
|
251
|
+
throw new Error("retry_after_ms must be greater than or equal to 0");
|
|
252
|
+
}
|
|
253
|
+
(0, project_db_queue_1.releaseExpiredProjectQueueLeases)(databasePath, {
|
|
254
|
+
queue_name: selectedQueue,
|
|
255
|
+
now_ms: currentNow,
|
|
256
|
+
});
|
|
257
|
+
const claimed = (0, project_db_queue_1.claimProjectQueueMessage)(databasePath, {
|
|
258
|
+
queue_name: selectedQueue,
|
|
259
|
+
lease_owner: input.lease_owner,
|
|
260
|
+
lease_ms: input.lease_ms,
|
|
261
|
+
now_ms: currentNow,
|
|
262
|
+
});
|
|
263
|
+
if (!claimed) {
|
|
264
|
+
return result({ status: "idle", queue_name: selectedQueue, queue_message: null });
|
|
265
|
+
}
|
|
266
|
+
let payload;
|
|
267
|
+
try {
|
|
268
|
+
payload = parsePayload(claimed.payload_json);
|
|
269
|
+
}
|
|
270
|
+
catch (err) {
|
|
271
|
+
const error = err instanceof Error ? err.message : String(err);
|
|
272
|
+
const partial = partialPayloadIds(claimed.payload_json);
|
|
273
|
+
const receipt = writeMaterializerReceipt(databasePath, {
|
|
274
|
+
payload: partial,
|
|
275
|
+
kind: "materializer-invalid-payload",
|
|
276
|
+
status: "rejected",
|
|
277
|
+
actor: input.lease_owner,
|
|
278
|
+
message: claimed,
|
|
279
|
+
error,
|
|
280
|
+
now_ms: currentNow,
|
|
281
|
+
receipts_path: input.receipts_path,
|
|
282
|
+
});
|
|
283
|
+
const dead = (0, project_db_queue_1.deadLetterProjectQueueMessage)(databasePath, {
|
|
284
|
+
queue_name: selectedQueue,
|
|
285
|
+
message_id: claimed.message_id,
|
|
286
|
+
lease_owner: input.lease_owner,
|
|
287
|
+
error,
|
|
288
|
+
now_ms: currentNow,
|
|
289
|
+
});
|
|
290
|
+
return result({
|
|
291
|
+
status: "dead_letter",
|
|
292
|
+
queue_name: selectedQueue,
|
|
293
|
+
queue_message: dead,
|
|
294
|
+
receipt,
|
|
295
|
+
error,
|
|
296
|
+
});
|
|
297
|
+
}
|
|
298
|
+
const leaseId = `materializer-${safeSegment(claimed.message_id)}-${claimed.attempt_count}`;
|
|
299
|
+
const currentBranchHash = readBranchSnapshotHash(databasePath, payload.project_id, payload.branch_id);
|
|
300
|
+
const baseHash = input.base_snapshot_hash ?? currentBranchHash ?? sha256File(databasePath);
|
|
301
|
+
let lease = null;
|
|
302
|
+
try {
|
|
303
|
+
lease = (0, project_db_events_1.acquireProjectWriterLease)(databasePath, {
|
|
304
|
+
project_id: payload.project_id,
|
|
305
|
+
branch_id: payload.branch_id,
|
|
306
|
+
lease_id: leaseId,
|
|
307
|
+
lease_owner: input.lease_owner,
|
|
308
|
+
base_snapshot_hash: baseHash,
|
|
309
|
+
lease_ms: input.lease_ms,
|
|
310
|
+
now_ms: currentNow,
|
|
311
|
+
});
|
|
312
|
+
if (currentBranchHash && currentBranchHash !== baseHash) {
|
|
313
|
+
const conflict = (0, project_db_events_1.commitProjectWriterLease)(databasePath, {
|
|
314
|
+
project_id: payload.project_id,
|
|
315
|
+
branch_id: payload.branch_id,
|
|
316
|
+
lease_id: leaseId,
|
|
317
|
+
lease_owner: input.lease_owner,
|
|
318
|
+
result_snapshot_hash: sha256File(databasePath),
|
|
319
|
+
now_ms: currentNow,
|
|
320
|
+
receipts_path: input.receipts_path,
|
|
321
|
+
});
|
|
322
|
+
const failed = (0, project_db_queue_1.failProjectQueueMessage)(databasePath, {
|
|
323
|
+
queue_name: selectedQueue,
|
|
324
|
+
message_id: claimed.message_id,
|
|
325
|
+
lease_owner: input.lease_owner,
|
|
326
|
+
error: `snapshot hash mismatch: current ${currentBranchHash}, base ${baseHash}`,
|
|
327
|
+
retry_after_ms: retryAfter,
|
|
328
|
+
now_ms: currentNow,
|
|
329
|
+
});
|
|
330
|
+
return result({
|
|
331
|
+
status: failStatus(failed, "conflict"),
|
|
332
|
+
queue_name: selectedQueue,
|
|
333
|
+
queue_message: failed,
|
|
334
|
+
payload,
|
|
335
|
+
lease: conflict.lease,
|
|
336
|
+
receipt: conflict.receipt,
|
|
337
|
+
error: failed.last_error,
|
|
338
|
+
});
|
|
339
|
+
}
|
|
340
|
+
const reducer = (0, project_db_events_1.applyProjectDbReducer)(databasePath, {
|
|
341
|
+
event_id: payload.event_id,
|
|
342
|
+
reducer_name: payload.reducer_name,
|
|
343
|
+
reducer_version: payload.reducer_version,
|
|
344
|
+
actor: input.lease_owner,
|
|
345
|
+
now_ms: currentNow,
|
|
346
|
+
receipts_path: input.receipts_path,
|
|
347
|
+
});
|
|
348
|
+
if (!reducer.applied && reducer.receipt.status === "rejected") {
|
|
349
|
+
tryReleaseWriterLease(databasePath, payload, leaseId, input.lease_owner, currentNow, input.receipts_path);
|
|
350
|
+
const failed = (0, project_db_queue_1.failProjectQueueMessage)(databasePath, {
|
|
351
|
+
queue_name: selectedQueue,
|
|
352
|
+
message_id: claimed.message_id,
|
|
353
|
+
lease_owner: input.lease_owner,
|
|
354
|
+
error: reducer.receipt.details_json,
|
|
355
|
+
retry_after_ms: retryAfter,
|
|
356
|
+
now_ms: currentNow,
|
|
357
|
+
});
|
|
358
|
+
return result({
|
|
359
|
+
status: failStatus(failed, "rejected"),
|
|
360
|
+
queue_name: selectedQueue,
|
|
361
|
+
queue_message: failed,
|
|
362
|
+
payload,
|
|
363
|
+
reducer,
|
|
364
|
+
lease,
|
|
365
|
+
receipt: reducer.receipt,
|
|
366
|
+
error: failed.last_error,
|
|
367
|
+
});
|
|
368
|
+
}
|
|
369
|
+
const commit = (0, project_db_events_1.commitProjectWriterLease)(databasePath, {
|
|
370
|
+
project_id: payload.project_id,
|
|
371
|
+
branch_id: payload.branch_id,
|
|
372
|
+
lease_id: leaseId,
|
|
373
|
+
lease_owner: input.lease_owner,
|
|
374
|
+
result_snapshot_hash: sha256File(databasePath),
|
|
375
|
+
now_ms: currentNow,
|
|
376
|
+
receipts_path: input.receipts_path,
|
|
377
|
+
});
|
|
378
|
+
if (!commit.committed) {
|
|
379
|
+
const failed = (0, project_db_queue_1.failProjectQueueMessage)(databasePath, {
|
|
380
|
+
queue_name: selectedQueue,
|
|
381
|
+
message_id: claimed.message_id,
|
|
382
|
+
lease_owner: input.lease_owner,
|
|
383
|
+
error: commit.receipt.details_json,
|
|
384
|
+
retry_after_ms: retryAfter,
|
|
385
|
+
now_ms: currentNow,
|
|
386
|
+
});
|
|
387
|
+
return result({
|
|
388
|
+
status: failStatus(failed, "conflict"),
|
|
389
|
+
queue_name: selectedQueue,
|
|
390
|
+
queue_message: failed,
|
|
391
|
+
payload,
|
|
392
|
+
reducer,
|
|
393
|
+
lease: commit.lease,
|
|
394
|
+
receipt: commit.receipt,
|
|
395
|
+
error: failed.last_error,
|
|
396
|
+
});
|
|
397
|
+
}
|
|
398
|
+
const acked = (0, project_db_queue_1.ackProjectQueueMessage)(databasePath, {
|
|
399
|
+
queue_name: selectedQueue,
|
|
400
|
+
message_id: claimed.message_id,
|
|
401
|
+
lease_owner: input.lease_owner,
|
|
402
|
+
now_ms: currentNow,
|
|
403
|
+
});
|
|
404
|
+
const repoRoot = findRepoRoot(databasePath, input.repo_root);
|
|
405
|
+
const config = (0, config_1.loadConfig)(repoRoot);
|
|
406
|
+
const snapshot = (0, project_db_snapshot_1.sealProjectDbSnapshot)(repoRoot, config);
|
|
407
|
+
return result({
|
|
408
|
+
status: reducer.applied ? "applied" : "duplicate",
|
|
409
|
+
queue_name: selectedQueue,
|
|
410
|
+
queue_message: acked,
|
|
411
|
+
payload,
|
|
412
|
+
reducer,
|
|
413
|
+
lease: commit.lease,
|
|
414
|
+
receipt: reducer.receipt,
|
|
415
|
+
snapshot,
|
|
416
|
+
});
|
|
417
|
+
}
|
|
418
|
+
catch (err) {
|
|
419
|
+
const error = err instanceof Error ? err.message : String(err);
|
|
420
|
+
if (lease) {
|
|
421
|
+
tryReleaseWriterLease(databasePath, payload, leaseId, input.lease_owner, currentNow, input.receipts_path);
|
|
422
|
+
}
|
|
423
|
+
const receipt = writeMaterializerReceipt(databasePath, {
|
|
424
|
+
payload,
|
|
425
|
+
kind: "materializer-error",
|
|
426
|
+
status: "rejected",
|
|
427
|
+
actor: input.lease_owner,
|
|
428
|
+
message: claimed,
|
|
429
|
+
error,
|
|
430
|
+
now_ms: currentNow,
|
|
431
|
+
receipts_path: input.receipts_path,
|
|
432
|
+
});
|
|
433
|
+
const failed = (0, project_db_queue_1.failProjectQueueMessage)(databasePath, {
|
|
434
|
+
queue_name: selectedQueue,
|
|
435
|
+
message_id: claimed.message_id,
|
|
436
|
+
lease_owner: input.lease_owner,
|
|
437
|
+
error,
|
|
438
|
+
retry_after_ms: retryAfter,
|
|
439
|
+
now_ms: currentNow,
|
|
440
|
+
});
|
|
441
|
+
return result({
|
|
442
|
+
status: failStatus(failed, "retry"),
|
|
443
|
+
queue_name: selectedQueue,
|
|
444
|
+
queue_message: failed,
|
|
445
|
+
payload,
|
|
446
|
+
lease,
|
|
447
|
+
receipt,
|
|
448
|
+
error,
|
|
449
|
+
});
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
function readProjectDbMaterializerStats(databasePath, input = {}) {
|
|
453
|
+
const selectedQueue = queueName(input.queue_name);
|
|
454
|
+
return {
|
|
455
|
+
queue_name: selectedQueue,
|
|
456
|
+
queue: (0, project_db_queue_1.readProjectQueueStats)(databasePath, {
|
|
457
|
+
queue_name: selectedQueue,
|
|
458
|
+
now_ms: input.now_ms,
|
|
459
|
+
}),
|
|
460
|
+
writer_leases: (0, project_db_events_1.readProjectWriterLeaseStats)(databasePath, { now_ms: input.now_ms }),
|
|
461
|
+
};
|
|
462
|
+
}
|
|
@@ -53,6 +53,90 @@ ON project_queue_message(queue_name, status, available_at_ms, created_at_ms, mes
|
|
|
53
53
|
CREATE INDEX IF NOT EXISTS project_queue_message_lease_idx
|
|
54
54
|
ON project_queue_message(queue_name, status, lease_deadline_ms, created_at_ms, message_id);
|
|
55
55
|
`;
|
|
56
|
+
const EVENTS_RECEIPTS_MIGRATION_SQL = `
|
|
57
|
+
CREATE TABLE IF NOT EXISTS project_event (
|
|
58
|
+
event_id TEXT PRIMARY KEY,
|
|
59
|
+
project_id TEXT NOT NULL,
|
|
60
|
+
branch_id TEXT NOT NULL,
|
|
61
|
+
event_type TEXT NOT NULL,
|
|
62
|
+
schema_version INTEGER NOT NULL CHECK(schema_version > 0),
|
|
63
|
+
idempotency_key TEXT NOT NULL,
|
|
64
|
+
payload_json TEXT NOT NULL,
|
|
65
|
+
payload_hash TEXT NOT NULL,
|
|
66
|
+
actor TEXT NOT NULL,
|
|
67
|
+
status TEXT NOT NULL CHECK(status IN ('received', 'validated', 'applied', 'rejected', 'dead_letter')),
|
|
68
|
+
occurred_at_ms INTEGER NOT NULL,
|
|
69
|
+
created_at_ms INTEGER NOT NULL,
|
|
70
|
+
updated_at_ms INTEGER NOT NULL,
|
|
71
|
+
last_error TEXT
|
|
72
|
+
) STRICT;
|
|
73
|
+
|
|
74
|
+
CREATE UNIQUE INDEX IF NOT EXISTS project_event_idempotency_unique
|
|
75
|
+
ON project_event(project_id, branch_id, idempotency_key);
|
|
76
|
+
|
|
77
|
+
CREATE INDEX IF NOT EXISTS project_event_branch_status_idx
|
|
78
|
+
ON project_event(project_id, branch_id, status, occurred_at_ms, event_id);
|
|
79
|
+
|
|
80
|
+
CREATE TABLE IF NOT EXISTS project_receipt (
|
|
81
|
+
receipt_id TEXT PRIMARY KEY,
|
|
82
|
+
project_id TEXT NOT NULL,
|
|
83
|
+
branch_id TEXT NOT NULL,
|
|
84
|
+
kind TEXT NOT NULL,
|
|
85
|
+
status TEXT NOT NULL CHECK(status IN ('applied', 'rejected', 'duplicate', 'conflict', 'replay', 'dead_letter')),
|
|
86
|
+
event_id TEXT,
|
|
87
|
+
idempotency_key TEXT,
|
|
88
|
+
payload_hash TEXT,
|
|
89
|
+
base_snapshot_hash TEXT,
|
|
90
|
+
result_snapshot_hash TEXT,
|
|
91
|
+
reducer_name TEXT,
|
|
92
|
+
reducer_version TEXT,
|
|
93
|
+
lease_id TEXT,
|
|
94
|
+
actor TEXT,
|
|
95
|
+
artifact_path TEXT NOT NULL,
|
|
96
|
+
artifact_hash TEXT NOT NULL,
|
|
97
|
+
details_json TEXT NOT NULL,
|
|
98
|
+
created_at_ms INTEGER NOT NULL
|
|
99
|
+
) STRICT;
|
|
100
|
+
|
|
101
|
+
CREATE INDEX IF NOT EXISTS project_receipt_branch_status_idx
|
|
102
|
+
ON project_receipt(project_id, branch_id, status, created_at_ms, receipt_id);
|
|
103
|
+
|
|
104
|
+
CREATE INDEX IF NOT EXISTS project_receipt_event_idx
|
|
105
|
+
ON project_receipt(event_id, created_at_ms, receipt_id)
|
|
106
|
+
WHERE event_id IS NOT NULL;
|
|
107
|
+
`;
|
|
108
|
+
const WRITER_LEASES_MIGRATION_SQL = `
|
|
109
|
+
CREATE TABLE IF NOT EXISTS project_branch_state (
|
|
110
|
+
project_id TEXT NOT NULL,
|
|
111
|
+
branch_id TEXT NOT NULL,
|
|
112
|
+
current_snapshot_hash TEXT NOT NULL,
|
|
113
|
+
updated_at_ms INTEGER NOT NULL,
|
|
114
|
+
PRIMARY KEY (project_id, branch_id)
|
|
115
|
+
) STRICT;
|
|
116
|
+
|
|
117
|
+
CREATE TABLE IF NOT EXISTS project_writer_lease (
|
|
118
|
+
project_id TEXT NOT NULL,
|
|
119
|
+
branch_id TEXT NOT NULL,
|
|
120
|
+
lease_id TEXT NOT NULL,
|
|
121
|
+
lease_owner TEXT NOT NULL,
|
|
122
|
+
base_snapshot_hash TEXT NOT NULL,
|
|
123
|
+
status TEXT NOT NULL CHECK(status IN ('active', 'committed', 'released', 'expired', 'conflict')),
|
|
124
|
+
lease_deadline_ms INTEGER NOT NULL,
|
|
125
|
+
result_snapshot_hash TEXT,
|
|
126
|
+
receipt_id TEXT,
|
|
127
|
+
created_at_ms INTEGER NOT NULL,
|
|
128
|
+
updated_at_ms INTEGER NOT NULL,
|
|
129
|
+
last_error TEXT,
|
|
130
|
+
PRIMARY KEY (project_id, branch_id, lease_id)
|
|
131
|
+
) STRICT;
|
|
132
|
+
|
|
133
|
+
CREATE UNIQUE INDEX IF NOT EXISTS project_writer_lease_active_unique
|
|
134
|
+
ON project_writer_lease(project_id, branch_id)
|
|
135
|
+
WHERE status = 'active';
|
|
136
|
+
|
|
137
|
+
CREATE INDEX IF NOT EXISTS project_writer_lease_status_idx
|
|
138
|
+
ON project_writer_lease(project_id, branch_id, status, lease_deadline_ms, lease_id);
|
|
139
|
+
`;
|
|
56
140
|
const BUILTIN_MIGRATIONS = [
|
|
57
141
|
{
|
|
58
142
|
ordinal: 1,
|
|
@@ -66,6 +150,18 @@ const BUILTIN_MIGRATIONS = [
|
|
|
66
150
|
filename: "002_mdkg_project_db_queue.sql",
|
|
67
151
|
sql: QUEUE_MIGRATION_SQL.trim(),
|
|
68
152
|
},
|
|
153
|
+
{
|
|
154
|
+
ordinal: 3,
|
|
155
|
+
key: "mdkg.project_db.events_receipts.v1",
|
|
156
|
+
filename: "003_mdkg_project_db_events_receipts.sql",
|
|
157
|
+
sql: EVENTS_RECEIPTS_MIGRATION_SQL.trim(),
|
|
158
|
+
},
|
|
159
|
+
{
|
|
160
|
+
ordinal: 4,
|
|
161
|
+
key: "mdkg.project_db.writer_leases.v1",
|
|
162
|
+
filename: "004_mdkg_project_db_writer_leases.sql",
|
|
163
|
+
sql: WRITER_LEASES_MIGRATION_SQL.trim(),
|
|
164
|
+
},
|
|
69
165
|
];
|
|
70
166
|
function loadDatabaseCtor() {
|
|
71
167
|
try {
|
|
@@ -207,7 +303,7 @@ function checkMigrationFiles(root, config) {
|
|
|
207
303
|
ok: errors.length === 0,
|
|
208
304
|
level: errors.length === 0 ? "ok" : "fail",
|
|
209
305
|
path: rel(root, layout.migrations),
|
|
210
|
-
detail: errors.length === 0 ? "migration files match mdkg-owned
|
|
306
|
+
detail: errors.length === 0 ? "migration files match mdkg-owned built-in migrations" : "migration file issues found",
|
|
211
307
|
errors,
|
|
212
308
|
warnings: [],
|
|
213
309
|
};
|
package/dist/init/AGENT_START.md
CHANGED
|
@@ -34,10 +34,13 @@ Agent operating prompt:
|
|
|
34
34
|
- Treat `.mdkg/db` as project application state; use `mdkg db init` to create
|
|
35
35
|
the generic scaffold and enable `db.enabled` without creating an active
|
|
36
36
|
runtime SQLite database. Use `mdkg db migrate` after init to create or update
|
|
37
|
-
the runtime SQLite database with mdkg-owned foundation
|
|
38
|
-
node:sqlite queue
|
|
39
|
-
infrastructure, not canonical event
|
|
40
|
-
|
|
37
|
+
the runtime SQLite database with mdkg-owned foundation plus internal local
|
|
38
|
+
node:sqlite queue, event/receipt/reducer, and writer lease/CAS foundation
|
|
39
|
+
migrations. Queue state is delivery infrastructure, not canonical event
|
|
40
|
+
history. Event rows are durable local project DB history; receipts, reducers,
|
|
41
|
+
writer leases, and materializers are internal local helper surfaces, and there
|
|
42
|
+
is no public `mdkg db queue`, `mdkg db event`, `mdkg db reducer`,
|
|
43
|
+
`mdkg db lease`, or `mdkg db materializer` CLI yet. Use `mdkg db verify` and `mdkg db stats` for
|
|
41
44
|
non-mutating health and summary receipts. Use `mdkg db snapshot seal` for
|
|
42
45
|
explicit sealed checkpoints,
|
|
43
46
|
`mdkg db snapshot verify/status` for checkpoint health, and
|
|
@@ -50,11 +50,15 @@ Project database commands:
|
|
|
50
50
|
active runtime SQLite database
|
|
51
51
|
- `mdkg db migrate` creates or updates the configured active runtime SQLite
|
|
52
52
|
database and applies mdkg-owned foundation plus internal local node:sqlite
|
|
53
|
-
queue foundation migrations
|
|
53
|
+
queue, event/receipt/reducer, and writer lease/CAS foundation migrations
|
|
54
54
|
- `mdkg db migrate` records migration order, checksums, and applied timestamps
|
|
55
55
|
in the configured migration table
|
|
56
56
|
- queue tables are durable local delivery state, not canonical event history;
|
|
57
57
|
there is no public `mdkg db queue` CLI yet
|
|
58
|
+
- event tables are durable local history for project DB state transitions;
|
|
59
|
+
receipts, typed reducers, writer leases, and materializers remain internal
|
|
60
|
+
helper surfaces in this release, with no public `mdkg db event`,
|
|
61
|
+
`mdkg db reducer`, `mdkg db lease`, or `mdkg db materializer` CLI yet
|
|
58
62
|
- `mdkg db verify` checks config, layout, runtime SQLite integrity, migration
|
|
59
63
|
metadata, receipt directory policy, and transient runtime files
|
|
60
64
|
- `mdkg db stats` reports table counts, database size, migration state,
|
package/dist/init/README.md
CHANGED
|
@@ -75,10 +75,13 @@ Fresh mdkg workspaces default to `index.backend: sqlite`; `.mdkg/index/mdkg.sqli
|
|
|
75
75
|
`.mdkg/index`. Run `mdkg db init` to create the generic scaffold, write
|
|
76
76
|
`.mdkg/db/project-db.json`, and enable `db.enabled`; it does not create an
|
|
77
77
|
active runtime SQLite database. Run `mdkg db migrate` after init to create or
|
|
78
|
-
update the active runtime SQLite database with mdkg-owned foundation
|
|
79
|
-
internal local node:sqlite queue
|
|
80
|
-
|
|
81
|
-
|
|
78
|
+
update the active runtime SQLite database with mdkg-owned foundation plus
|
|
79
|
+
internal local node:sqlite queue, event/receipt/reducer, and writer lease/CAS
|
|
80
|
+
foundation migrations. Queue state is delivery infrastructure, not canonical
|
|
81
|
+
event history. Event rows are durable local project DB history; receipts,
|
|
82
|
+
reducers, writer leases, and materializers are internal local helper surfaces,
|
|
83
|
+
and there is no public `mdkg db queue`, `mdkg db event`, `mdkg db reducer`,
|
|
84
|
+
`mdkg db lease`, or `mdkg db materializer` CLI yet. Use `mdkg db verify` for non-mutating health checks and
|
|
82
85
|
`mdkg db stats` for table counts, DB size, migration state, and receipt-file
|
|
83
86
|
counts. Use `mdkg db snapshot seal` to create an opt-in sealed checkpoint under
|
|
84
87
|
`.mdkg/db/state`, then use `mdkg db snapshot verify/status` for integrity and
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"schema_version": 1,
|
|
3
3
|
"tool": "mdkg",
|
|
4
|
-
"mdkg_version": "0.1.
|
|
4
|
+
"mdkg_version": "0.1.10",
|
|
5
5
|
"files": [
|
|
6
6
|
{
|
|
7
7
|
"path": ".mdkg/config.json",
|
|
@@ -61,7 +61,7 @@
|
|
|
61
61
|
{
|
|
62
62
|
"path": ".mdkg/README.md",
|
|
63
63
|
"category": "mdkg_doc",
|
|
64
|
-
"sha256": "
|
|
64
|
+
"sha256": "0b5e7fa852aa71616ac108b2af3fb21b9ab66ee1ea69cb75539ba1df80be1072"
|
|
65
65
|
},
|
|
66
66
|
{
|
|
67
67
|
"path": ".mdkg/skills/build-pack-and-execute-task/SKILL.md",
|
|
@@ -186,7 +186,7 @@
|
|
|
186
186
|
{
|
|
187
187
|
"path": "AGENT_START.md",
|
|
188
188
|
"category": "startup_doc",
|
|
189
|
-
"sha256": "
|
|
189
|
+
"sha256": "1e9def4cf02de6eecd164cff7855829dc3632009a2aa801eccb67b89bad4a570"
|
|
190
190
|
},
|
|
191
191
|
{
|
|
192
192
|
"path": "AGENTS.md",
|
|
@@ -201,7 +201,7 @@
|
|
|
201
201
|
{
|
|
202
202
|
"path": "CLI_COMMAND_MATRIX.md",
|
|
203
203
|
"category": "startup_doc",
|
|
204
|
-
"sha256": "
|
|
204
|
+
"sha256": "95bc386d88817ca597ae4f0fe4fa2d904843227a81147d606f54a32b93694c83"
|
|
205
205
|
},
|
|
206
206
|
{
|
|
207
207
|
"path": "llms.txt",
|