mdkg 0.1.9 → 0.2.0

package/dist/core/project_db_materializer.js

@@ -0,0 +1,462 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PROJECT_DB_MATERIALIZER_SCHEMA_VERSION = exports.PROJECT_DB_MATERIALIZER_KIND = exports.PROJECT_DB_MATERIALIZER_QUEUE = void 0;
+exports.enqueueProjectDbMaterialization = enqueueProjectDbMaterialization;
+exports.runNextProjectDbMaterializer = runNextProjectDbMaterializer;
+exports.readProjectDbMaterializerStats = readProjectDbMaterializerStats;
+const crypto_1 = __importDefault(require("crypto"));
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const config_1 = require("./config");
+const project_db_queue_1 = require("./project_db_queue");
+const project_db_events_1 = require("./project_db_events");
+const project_db_snapshot_1 = require("./project_db_snapshot");
+exports.PROJECT_DB_MATERIALIZER_QUEUE = "project-db.materialize";
+exports.PROJECT_DB_MATERIALIZER_KIND = "project-db.materialize";
+exports.PROJECT_DB_MATERIALIZER_SCHEMA_VERSION = 1;
+function loadDatabaseCtor() {
+    try {
+        const loaded = require("node:sqlite");
+        if (!loaded.DatabaseSync) {
+            throw new Error("node:sqlite DatabaseSync is unavailable");
+        }
+        return loaded.DatabaseSync;
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        throw new Error(`node:sqlite is required for mdkg project DB materializers: ${message}`);
+    }
+}
+function nowMs(input) {
+    if (input !== undefined) {
+        assertInteger(input, "now_ms");
+        return input;
+    }
+    return Date.now();
+}
+function assertNonEmpty(value, field) {
+    if (typeof value !== "string" || value.trim() === "") {
+        throw new Error(`${field} must be a non-empty string`);
+    }
+}
+function assertInteger(value, field) {
+    if (!Number.isInteger(value)) {
+        throw new Error(`${field} must be an integer`);
+    }
+}
+function assertPositiveInteger(value, field) {
+    assertInteger(value, field);
+    if (value <= 0) {
+        throw new Error(`${field} must be greater than 0`);
+    }
+}
+function sha256File(filePath) {
+    return `sha256:${crypto_1.default.createHash("sha256").update(fs_1.default.readFileSync(filePath)).digest("hex")}`;
+}
+function safeSegment(value) {
+    return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "materializer";
+}
+function isObject(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function stringField(value, field) {
+    const raw = value[field];
+    if (typeof raw !== "string" || raw.trim() === "") {
+        throw new Error(`materializer payload ${field} must be a non-empty string`);
+    }
+    return raw;
+}
+function parsePayload(payloadJson) {
+    let parsed;
+    try {
+        parsed = JSON.parse(payloadJson);
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        throw new Error(`materializer payload must be valid JSON: ${message}`);
+    }
+    if (!isObject(parsed)) {
+        throw new Error("materializer payload must be an object");
+    }
+    if (parsed.kind !== exports.PROJECT_DB_MATERIALIZER_KIND) {
+        throw new Error(`materializer payload kind must be ${exports.PROJECT_DB_MATERIALIZER_KIND}`);
+    }
+    if (parsed.schema_version !== exports.PROJECT_DB_MATERIALIZER_SCHEMA_VERSION) {
+        throw new Error(`materializer payload schema_version must be ${exports.PROJECT_DB_MATERIALIZER_SCHEMA_VERSION}`);
+    }
+    const reducerName = stringField(parsed, "reducer_name");
+    if (reducerName !== "project_meta.set") {
+        throw new Error(`unsupported materializer reducer_name: ${reducerName}`);
+    }
+    const reducerVersion = stringField(parsed, "reducer_version");
+    if (reducerVersion !== "v1") {
+        throw new Error(`unsupported materializer reducer_version: ${reducerVersion}`);
+    }
+    return {
+        kind: exports.PROJECT_DB_MATERIALIZER_KIND,
+        schema_version: exports.PROJECT_DB_MATERIALIZER_SCHEMA_VERSION,
+        project_id: stringField(parsed, "project_id"),
+        branch_id: stringField(parsed, "branch_id"),
+        event_id: stringField(parsed, "event_id"),
+        reducer_name: reducerName,
+        reducer_version: reducerVersion,
+    };
+}
+function partialPayloadIds(payloadJson) {
+    try {
+        const parsed = JSON.parse(payloadJson);
+        if (isObject(parsed)) {
+            return {
+                project_id: typeof parsed.project_id === "string" && parsed.project_id.trim() !== "" ? parsed.project_id : "unknown",
+                branch_id: typeof parsed.branch_id === "string" && parsed.branch_id.trim() !== "" ? parsed.branch_id : "unknown",
+                event_id: typeof parsed.event_id === "string" && parsed.event_id.trim() !== "" ? parsed.event_id : null,
+            };
+        }
+    }
+    catch {
+        // fall through to unknown identifiers for rejected payload receipts
+    }
+    return { project_id: "unknown", branch_id: "unknown", event_id: null };
+}
+function findRepoRoot(databasePath, explicitRoot) {
+    if (explicitRoot) {
+        return path_1.default.resolve(explicitRoot);
+    }
+    let current = path_1.default.dirname(path_1.default.resolve(databasePath));
+    while (true) {
+        if (fs_1.default.existsSync(path_1.default.join(current, ".mdkg", "config.json"))) {
+            return current;
+        }
+        const parent = path_1.default.dirname(current);
+        if (parent === current) {
+            throw new Error("repo_root is required when databasePath is not inside an mdkg repo");
+        }
+        current = parent;
+    }
+}
+function withDb(databasePath, fn) {
+    const DatabaseSync = loadDatabaseCtor();
+    const db = new DatabaseSync(databasePath);
+    try {
+        db.exec("PRAGMA foreign_keys = ON;");
+        return fn(db);
+    }
+    finally {
+        db.close();
+    }
+}
+function readBranchSnapshotHash(databasePath, projectId, branchId) {
+    return withDb(databasePath, (db) => {
+        const row = db
+            .prepare("SELECT current_snapshot_hash FROM project_branch_state WHERE project_id = ? AND branch_id = ?")
+            .get(projectId, branchId);
+        return row?.current_snapshot_hash === undefined || row.current_snapshot_hash === null
+            ? null
+            : String(row.current_snapshot_hash);
+    });
+}
+function queueName(input) {
+    if (input !== undefined) {
+        assertNonEmpty(input, "queue_name");
+        return input;
+    }
+    return exports.PROJECT_DB_MATERIALIZER_QUEUE;
+}
+function result(input) {
+    return {
+        status: input.status,
+        queue_name: input.queue_name,
+        queue_message: input.queue_message,
+        payload: input.payload ?? null,
+        reducer: input.reducer ?? null,
+        lease: input.lease ?? null,
+        receipt: input.receipt ?? null,
+        snapshot: input.snapshot ?? null,
+        error: input.error ?? null,
+    };
+}
+function failStatus(message, fallback) {
+    return message.status === "dead_letter" ? "dead_letter" : fallback;
+}
+function writeMaterializerReceipt(databasePath, input) {
+    return (0, project_db_events_1.writeProjectDbReceipt)(databasePath, {
+        project_id: input.payload.project_id,
+        branch_id: input.payload.branch_id,
+        kind: input.kind,
+        status: input.status,
+        event_id: input.payload.event_id,
+        actor: input.actor,
+        details: {
+            message_id: input.message.message_id,
+            queue_name: input.message.queue_name,
+            error: input.error,
+        },
+        now_ms: input.now_ms,
+        receipts_path: input.receipts_path,
+    });
+}
+function tryReleaseWriterLease(databasePath, payload, leaseId, leaseOwner, now, receiptsPath) {
+    try {
+        (0, project_db_events_1.releaseProjectWriterLease)(databasePath, {
+            project_id: payload.project_id,
+            branch_id: payload.branch_id,
+            lease_id: leaseId,
+            lease_owner: leaseOwner,
+            now_ms: now,
+            receipts_path: receiptsPath,
+        });
+    }
+    catch {
+        // The lease may already be committed, conflicted, released, or absent.
+    }
+}
+function enqueueProjectDbMaterialization(databasePath, input) {
+    assertNonEmpty(input.message_id, "message_id");
+    assertNonEmpty(input.project_id, "project_id");
+    assertNonEmpty(input.branch_id, "branch_id");
+    assertNonEmpty(input.event_id, "event_id");
+    const selectedQueue = queueName(input.queue_name);
+    const payload = {
+        kind: exports.PROJECT_DB_MATERIALIZER_KIND,
+        schema_version: exports.PROJECT_DB_MATERIALIZER_SCHEMA_VERSION,
+        project_id: input.project_id,
+        branch_id: input.branch_id,
+        event_id: input.event_id,
+        reducer_name: input.reducer_name,
+        reducer_version: input.reducer_version,
+    };
+    return (0, project_db_queue_1.enqueueProjectQueueMessage)(databasePath, {
+        queue_name: selectedQueue,
+        message_id: input.message_id,
+        dedupe_key: input.dedupe_key === undefined
+            ? `${input.project_id}:${input.branch_id}:${input.event_id}:${input.reducer_name}:${input.reducer_version}`
+            : input.dedupe_key,
+        payload,
+        available_at_ms: input.available_at_ms,
+        max_attempts: input.max_attempts,
+        now_ms: input.now_ms,
+    });
+}
+function runNextProjectDbMaterializer(databasePath, input) {
+    assertNonEmpty(input.lease_owner, "lease_owner");
+    assertPositiveInteger(input.lease_ms, "lease_ms");
+    const selectedQueue = queueName(input.queue_name);
+    const currentNow = nowMs(input.now_ms);
+    const retryAfter = input.retry_after_ms ?? 0;
+    assertInteger(retryAfter, "retry_after_ms");
+    if (retryAfter < 0) {
+        throw new Error("retry_after_ms must be greater than or equal to 0");
+    }
+    (0, project_db_queue_1.releaseExpiredProjectQueueLeases)(databasePath, {
+        queue_name: selectedQueue,
+        now_ms: currentNow,
+    });
+    const claimed = (0, project_db_queue_1.claimProjectQueueMessage)(databasePath, {
+        queue_name: selectedQueue,
+        lease_owner: input.lease_owner,
+        lease_ms: input.lease_ms,
+        now_ms: currentNow,
+    });
+    if (!claimed) {
+        return result({ status: "idle", queue_name: selectedQueue, queue_message: null });
+    }
+    let payload;
+    try {
+        payload = parsePayload(claimed.payload_json);
+    }
+    catch (err) {
+        const error = err instanceof Error ? err.message : String(err);
+        const partial = partialPayloadIds(claimed.payload_json);
+        const receipt = writeMaterializerReceipt(databasePath, {
+            payload: partial,
+            kind: "materializer-invalid-payload",
+            status: "rejected",
+            actor: input.lease_owner,
+            message: claimed,
+            error,
+            now_ms: currentNow,
+            receipts_path: input.receipts_path,
+        });
+        const dead = (0, project_db_queue_1.deadLetterProjectQueueMessage)(databasePath, {
+            queue_name: selectedQueue,
+            message_id: claimed.message_id,
+            lease_owner: input.lease_owner,
+            error,
+            now_ms: currentNow,
+        });
+        return result({
+            status: "dead_letter",
+            queue_name: selectedQueue,
+            queue_message: dead,
+            receipt,
+            error,
+        });
+    }
+    const leaseId = `materializer-${safeSegment(claimed.message_id)}-${claimed.attempt_count}`;
+    const currentBranchHash = readBranchSnapshotHash(databasePath, payload.project_id, payload.branch_id);
+    const baseHash = input.base_snapshot_hash ?? currentBranchHash ?? sha256File(databasePath);
+    let lease = null;
+    try {
+        lease = (0, project_db_events_1.acquireProjectWriterLease)(databasePath, {
+            project_id: payload.project_id,
+            branch_id: payload.branch_id,
+            lease_id: leaseId,
+            lease_owner: input.lease_owner,
+            base_snapshot_hash: baseHash,
+            lease_ms: input.lease_ms,
+            now_ms: currentNow,
+        });
+        if (currentBranchHash && currentBranchHash !== baseHash) {
+            const conflict = (0, project_db_events_1.commitProjectWriterLease)(databasePath, {
+                project_id: payload.project_id,
+                branch_id: payload.branch_id,
+                lease_id: leaseId,
+                lease_owner: input.lease_owner,
+                result_snapshot_hash: sha256File(databasePath),
+                now_ms: currentNow,
+                receipts_path: input.receipts_path,
+            });
+            const failed = (0, project_db_queue_1.failProjectQueueMessage)(databasePath, {
+                queue_name: selectedQueue,
+                message_id: claimed.message_id,
+                lease_owner: input.lease_owner,
+                error: `snapshot hash mismatch: current ${currentBranchHash}, base ${baseHash}`,
+                retry_after_ms: retryAfter,
+                now_ms: currentNow,
+            });
+            return result({
+                status: failStatus(failed, "conflict"),
+                queue_name: selectedQueue,
+                queue_message: failed,
+                payload,
+                lease: conflict.lease,
+                receipt: conflict.receipt,
+                error: failed.last_error,
+            });
+        }
+        const reducer = (0, project_db_events_1.applyProjectDbReducer)(databasePath, {
+            event_id: payload.event_id,
+            reducer_name: payload.reducer_name,
+            reducer_version: payload.reducer_version,
+            actor: input.lease_owner,
+            now_ms: currentNow,
+            receipts_path: input.receipts_path,
+        });
+        if (!reducer.applied && reducer.receipt.status === "rejected") {
+            tryReleaseWriterLease(databasePath, payload, leaseId, input.lease_owner, currentNow, input.receipts_path);
+            const failed = (0, project_db_queue_1.failProjectQueueMessage)(databasePath, {
+                queue_name: selectedQueue,
+                message_id: claimed.message_id,
+                lease_owner: input.lease_owner,
+                error: reducer.receipt.details_json,
+                retry_after_ms: retryAfter,
+                now_ms: currentNow,
+            });
+            return result({
+                status: failStatus(failed, "rejected"),
+                queue_name: selectedQueue,
+                queue_message: failed,
+                payload,
+                reducer,
+                lease,
+                receipt: reducer.receipt,
+                error: failed.last_error,
+            });
+        }
+        const commit = (0, project_db_events_1.commitProjectWriterLease)(databasePath, {
+            project_id: payload.project_id,
+            branch_id: payload.branch_id,
+            lease_id: leaseId,
+            lease_owner: input.lease_owner,
+            result_snapshot_hash: sha256File(databasePath),
+            now_ms: currentNow,
+            receipts_path: input.receipts_path,
+        });
+        if (!commit.committed) {
+            const failed = (0, project_db_queue_1.failProjectQueueMessage)(databasePath, {
+                queue_name: selectedQueue,
+                message_id: claimed.message_id,
+                lease_owner: input.lease_owner,
+                error: commit.receipt.details_json,
+                retry_after_ms: retryAfter,
+                now_ms: currentNow,
+            });
+            return result({
+                status: failStatus(failed, "conflict"),
+                queue_name: selectedQueue,
+                queue_message: failed,
+                payload,
+                reducer,
+                lease: commit.lease,
+                receipt: commit.receipt,
+                error: failed.last_error,
+            });
+        }
+        const acked = (0, project_db_queue_1.ackProjectQueueMessage)(databasePath, {
+            queue_name: selectedQueue,
+            message_id: claimed.message_id,
+            lease_owner: input.lease_owner,
+            now_ms: currentNow,
+        });
+        const repoRoot = findRepoRoot(databasePath, input.repo_root);
+        const config = (0, config_1.loadConfig)(repoRoot);
+        const snapshot = (0, project_db_snapshot_1.sealProjectDbSnapshot)(repoRoot, config);
+        return result({
+            status: reducer.applied ? "applied" : "duplicate",
+            queue_name: selectedQueue,
+            queue_message: acked,
+            payload,
+            reducer,
+            lease: commit.lease,
+            receipt: reducer.receipt,
+            snapshot,
+        });
+    }
+    catch (err) {
+        const error = err instanceof Error ? err.message : String(err);
+        if (lease) {
+            tryReleaseWriterLease(databasePath, payload, leaseId, input.lease_owner, currentNow, input.receipts_path);
+        }
+        const receipt = writeMaterializerReceipt(databasePath, {
+            payload,
+            kind: "materializer-error",
+            status: "rejected",
+            actor: input.lease_owner,
+            message: claimed,
+            error,
+            now_ms: currentNow,
+            receipts_path: input.receipts_path,
+        });
+        const failed = (0, project_db_queue_1.failProjectQueueMessage)(databasePath, {
+            queue_name: selectedQueue,
+            message_id: claimed.message_id,
+            lease_owner: input.lease_owner,
+            error,
+            retry_after_ms: retryAfter,
+            now_ms: currentNow,
+        });
+        return result({
+            status: failStatus(failed, "retry"),
+            queue_name: selectedQueue,
+            queue_message: failed,
+            payload,
+            lease,
+            receipt,
+            error,
+        });
+    }
+}
+function readProjectDbMaterializerStats(databasePath, input = {}) {
+    const selectedQueue = queueName(input.queue_name);
+    return {
+        queue_name: selectedQueue,
+        queue: (0, project_db_queue_1.readProjectQueueStats)(databasePath, {
+            queue_name: selectedQueue,
+            now_ms: input.now_ms,
+        }),
+        writer_leases: (0, project_db_events_1.readProjectWriterLeaseStats)(databasePath, { now_ms: input.now_ms }),
+    };
+}

package/dist/core/project_db_migrations.js

@@ -53,6 +53,24 @@ ON project_queue_message(queue_name, status, available_at_ms, created_at_ms, mes
 CREATE INDEX IF NOT EXISTS project_queue_message_lease_idx
 ON project_queue_message(queue_name, status, lease_deadline_ms, created_at_ms, message_id);
 `;
+const QUEUE_CONTROL_MIGRATION_SQL = `
+CREATE TABLE IF NOT EXISTS project_queue (
+  queue_name TEXT PRIMARY KEY,
+  status TEXT NOT NULL CHECK(status IN ('active', 'paused')),
+  paused_reason TEXT,
+  created_at_ms INTEGER NOT NULL,
+  updated_at_ms INTEGER NOT NULL
+) STRICT;
+
+INSERT OR IGNORE INTO project_queue (queue_name, status, paused_reason, created_at_ms, updated_at_ms)
+SELECT DISTINCT
+  queue_name,
+  'active',
+  NULL,
+  CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER),
+  CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)
+FROM project_queue_message;
+`;
 const EVENTS_RECEIPTS_MIGRATION_SQL = `
 CREATE TABLE IF NOT EXISTS project_event (
   event_id TEXT PRIMARY KEY,
@@ -162,6 +180,12 @@ const BUILTIN_MIGRATIONS = [
         filename: "004_mdkg_project_db_writer_leases.sql",
         sql: WRITER_LEASES_MIGRATION_SQL.trim(),
     },
+    {
+        ordinal: 5,
+        key: "mdkg.project_db.queue_control.v1",
+        filename: "005_mdkg_project_db_queue_control.sql",
+        sql: QUEUE_CONTROL_MIGRATION_SQL.trim(),
+    },
 ];
 function loadDatabaseCtor() {
     try {