@playcraft/cli 0.0.58 → 0.1.0

package/dist/workspace-runtime/persistence/journal.js

@@ -1,10 +1,27 @@
-import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
+import { copyFile, mkdir, readFile, writeFile } from 'node:fs/promises';
+import { existsSync } from 'node:fs';
 import { dirname, join } from 'node:path';
-import { workspaceRegistrationSchema, checkpointRecordSchema, operationRecordSchema, pendingSourceEventSchema, sourceEventKey, sourceEventReceiptSchema, terminalRecordSchema, } from '@playcraft/common/cloud-workspace/runtime';
+import { workspaceRegistrationSchema, checkpointRecordSchema, operationRecordSchema, pendingSourceEventSchema, replayPinResultSchema, retainedRangeHasGap, sourceAckSchema, sourceEventKey, sourceEventReceiptSchema, terminalRecordSchema, } from '@playcraft/common/cloud-workspace/runtime';
+import { SqliteWorkerClient } from './sqlite-client.js';
+export const RUNTIME_STATE_FILE = 'runtime-state.sqlite';
+export const LEGACY_JOURNAL_FILE = 'journal.json';
+export const LEGACY_JOURNAL_SENTINEL = 'PLAYCRAFT_RUNTIME_STATE_SQLITE\n';
 export class RuntimeJournal {
     filePath;
-    writes = Promise.resolve();
+    legacyPath;
+    client;
+    listeners = new Set();
+    maintenance;
+    engineInfo;
+    epochs = [];
+    readToken = '';
+    inboxPending = 0;
     locks = new Map();
+    snapshot = emptySnapshot();
+    constructor(stateDir) {
+        this.filePath = join(stateDir, RUNTIME_STATE_FILE);
+        this.legacyPath = join(stateDir, LEGACY_JOURNAL_FILE);
+    }
     async exclusive(key, action) {
         const previous = this.locks.get(key) ?? Promise.resolve();
         let release;
@@ -22,93 +39,52 @@ export class RuntimeJournal {
                 this.locks.delete(key);
         }
     }
-    snapshot = {
-        workspaces: {},
-        workspaceFileRevisions: {},
-        operations: {},
-        receipts: {},
-        events: {},
-        sessions: {},
-        terminals: {},
-        checkpoints: {},
-        fileRevisions: {},
-        approvals: {},
-        managedWrites: {},
-        executionGroup: { stopConfirmed: false, replaced: false },
-        lastSourceSequence: 0,
-        acceptedLeaseFence: 0,
-        destroyed: false,
-    };
-    constructor(stateDir) {
-        this.filePath = join(stateDir, 'journal.json');
-    }
     async load() {
-        try {
-            const parsed = JSON.parse(await readFile(this.filePath, 'utf8'));
-            this.snapshot = {
-                workspaces: Object.fromEntries(Object.entries(parsed.workspaces ?? {}).map(([key, value]) => [key, workspaceRegistrationSchema.parse(value)])),
-                workspaceFileRevisions: parsed.workspaceFileRevisions ?? {},
-                operations: Object.fromEntries(Object.entries(parsed.operations ?? {}).map(([key, value]) => [
-                    key,
-                    operationRecordSchema.parse(value),
-                ])),
-                receipts: Object.fromEntries(Object.entries(parsed.receipts ?? {}).map(([key, value]) => [
-                    key,
-                    sourceEventReceiptSchema.parse(value),
-                ])),
-                events: Object.fromEntries(Object.entries(parsed.events ?? {}).map(([key, value]) => [
-                    key,
-                    pendingSourceEventSchema.parse(value),
-                ])),
-                sessions: Object.fromEntries(Object.entries(parsed.sessions ?? {}).map(([key, value]) => [
-                    key,
-                    {
-                        ...value,
-                        recoverability: value.recoverability === 'history_only' ? 'history_only' : 'native',
-                    },
-                ])),
-                terminals: Object.fromEntries(Object.entries(parsed.terminals ?? {}).map(([key, value]) => [
-                    key,
-                    terminalRecordSchema.parse(value),
-                ])),
-                checkpoints: Object.fromEntries(Object.entries(parsed.checkpoints ?? {}).map(([key, value]) => [
-                    key,
-                    checkpointRecordSchema.parse(value),
-                ])),
-                fileRevisions: parsed.fileRevisions ?? {},
-                managedWrites: sanitizeManagedWrites(parsed.managedWrites),
-                fileCheckpointId: typeof parsed.fileCheckpointId === 'string' ? parsed.fileCheckpointId : undefined,
-                consistentCheckpointId: typeof parsed.consistentCheckpointId === 'string' ? parsed.consistentCheckpointId : undefined,
-                approvals: Object.fromEntries(Object.entries(parsed.approvals ?? {})
-                    .filter(([, value]) => isStoredApproval(value))
-                    .map(([key, value]) => [key, value])),
-                executionGroup: {
-                    pid: integerOrUndefined(parsed.executionGroup?.pid),
-                    startedAt: typeof parsed.executionGroup?.startedAt === 'string'
-                        ? parsed.executionGroup.startedAt
-                        : undefined,
-                    previousPid: integerOrUndefined(parsed.executionGroup?.previousPid),
-                    stopConfirmed: parsed.executionGroup?.stopConfirmed === true,
-                    replaced: parsed.executionGroup?.replaced === true,
-                },
-                runtimeEpoch: integerOrUndefined(parsed.runtimeEpoch),
-                lastSourceSequence: parsed.lastSourceSequence ?? 0,
-                acceptedLeaseFence: parsed.acceptedLeaseFence ?? highestFence(parsed.operations ?? {}),
-                destroyed: parsed.destroyed === true,
-            };
-        }
-        catch (error) {
-            if (error.code === 'ENOENT')
-                return;
-            throw error;
-        }
+        await mkdir(dirname(this.filePath), { recursive: true, mode: 0o700 });
+        const legacy = await readLegacy(this.legacyPath, this.filePath);
+        const started = await SqliteWorkerClient.open(this.filePath, legacy?.value ?? null);
+        this.client = started.client;
+        this.apply(started.opened);
+        if (legacy?.readable)
+            await writeFile(this.legacyPath, LEGACY_JOURNAL_SENTINEL, { mode: 0o600 });
+        this.maintenance = setInterval(() => {
+            void this.client?.request('maintain').catch(() => undefined);
+        }, 30_000);
+        this.maintenance.unref?.();
+    }
+    async close() {
+        if (this.maintenance)
+            clearInterval(this.maintenance);
+        await this.client?.close();
+        this.client = undefined;
+    }
+    engine() {
+        if (!this.engineInfo)
+            throw new Error('sqlite_engine_unavailable');
+        return this.engineInfo;
+    }
+    sourceReadToken() {
+        return this.readToken;
+    }
+    pendingInbox() {
+        return this.inboxPending;
+    }
+    epochsSnapshot() {
+        return this.epochs.map((epoch) => ({ ...epoch }));
+    }
+    onSourceEvent(listener) {
+        this.listeners.add(listener);
+        return () => this.listeners.delete(listener);
     }
     getWorkspace(workspaceId) {
         return this.snapshot.workspaces[workspaceId];
     }
     async putWorkspace(binding) {
-        this.snapshot.workspaces[binding.workspaceId] = workspaceRegistrationSchema.parse(binding);
-        await this.flush();
+        const parsed = workspaceRegistrationSchema.parse(binding);
+        await this.run('upsertRecord', { bucket: 'workspaces', key: parsed.workspaceId, body: parsed });
+    }
+    async forgetWorkspaces() {
+        await this.run('forgetWorkspaces', {});
     }
     findSessionCreation(sessionId) {
         return Object.values(this.snapshot.operations).find((item) => item.kind === 'session.create' && item.sessionId === sessionId);
@@ -147,12 +123,10 @@ export class RuntimeJournal {
         return this.snapshot.consistentCheckpointId;
     }
     async setFileCheckpointId(checkpointId) {
-        this.snapshot.fileCheckpointId = checkpointId;
-        await this.flush();
+        await this.run('setCheckpointId', { field: 'file', id: checkpointId });
     }
     async setConsistentCheckpointId(checkpointId) {
-        this.snapshot.consistentCheckpointId = checkpointId;
-        await this.flush();
+        await this.run('setCheckpointId', { field: 'consistent', id: checkpointId });
     }
     listManagedWrites(workspaceId) {
         return Object.values(this.snapshot.managedWrites).filter((item) => !workspaceId || item.workspaceId === workspaceId);
@@ -161,8 +135,7 @@ export class RuntimeJournal {
         return this.snapshot.managedWrites[writeId];
     }
     async putManagedWrite(record) {
-        this.snapshot.managedWrites[record.writeId] = record;
-        await this.flush();
+        await this.run('upsertRecord', { bucket: 'managed_writes', key: record.writeId, body: record });
     }
     fileRevisions(workspaceId) {
         return { ...(workspaceId ? this.snapshot.workspaceFileRevisions[workspaceId] ?? {} : this.snapshot.fileRevisions) };
@@ -186,165 +159,205 @@ export class RuntimeJournal {
         return { ...this.snapshot.executionGroup };
     }
     getApproval(sessionId, approvalId) {
-        return this.snapshot.approvals[approvalKey(sessionId, approvalId)];
+        return this.snapshot.approvals[`${sessionId}:${approvalId}`];
     }
     listApprovals() {
         return Object.values(this.snapshot.approvals);
     }
     async putApproval(approval) {
-        this.snapshot.approvals[approvalKey(approval.sessionId, approval.approvalId)] = approval;
-        await this.flush();
+        await this.run('upsertRecord', { bucket: 'approvals', key: `${approval.sessionId}:${approval.approvalId}`, body: approval });
     }
     async noteExecutionGroup(group) {
-        this.snapshot.executionGroup = {
-            ...this.snapshot.executionGroup,
-            pid: group.pid,
-            startedAt: group.startedAt,
-        };
-        await this.flush();
+        await this.run('noteExecutionGroup', { group });
     }
     async writeExecutionEpoch(input) {
-        this.snapshot.runtimeEpoch = input.runtimeEpoch;
-        this.snapshot.executionGroup = {
-            pid: input.pid,
-            startedAt: input.startedAt,
-            previousPid: input.previousPid,
-            stopConfirmed: input.stopConfirmed,
-            replaced: true,
-        };
-        for (const [key, approval] of Object.entries(this.snapshot.approvals)) {
-            if (approval.status === 'pending') {
-                this.snapshot.approvals[key] = { ...approval, status: 'expired' };
-            }
-        }
-        await this.flush();
+        await this.run('writeExecutionEpoch', input);
     }
     async setExecutionStopConfirmed(stopConfirmed) {
-        this.snapshot.executionGroup = { ...this.snapshot.executionGroup, stopConfirmed };
-        await this.flush();
+        await this.run('setStopConfirmed', { stopConfirmed });
+    }
+    async bindRuntime(input) {
+        await this.run('bindRuntime', input);
     }
     listEvents(afterSequence = 0) {
         return Object.values(this.snapshot.events)
             .filter((event) => event.source.sourceSequence > afterSequence)
+            .sort((left, right) => left.source.sourceSequence - right.source.sourceSequence || left.source.runtimeEpoch - right.source.runtimeEpoch);
+    }
+    readSourcePage(input) {
+        const epoch = this.epochs.find((item) => item.runtime_id === input.runtimeId && item.generation === input.generation && item.runtime_epoch === input.runtimeEpoch);
+        const retainedFrom = epoch?.retained_from ?? 1;
+        const highWater = epoch?.local_durable_through ?? 0;
+        const coveredThrough = input.after;
+        if (retainedRangeHasGap(retainedFrom, coveredThrough)) {
+            return { ok: false, code: 'source_gap', retainedFrom, highWater, coveredThrough, ...identity(input) };
+        }
+        const matched = Object.values(this.snapshot.events)
+            .filter((event) => event.source.runtimeId === input.runtimeId &&
+            event.source.generation === input.generation &&
+            event.source.runtimeEpoch === input.runtimeEpoch &&
+            event.source.sourceSequence > input.after)
             .sort((left, right) => left.source.sourceSequence - right.source.sourceSequence);
+        const first = matched[0];
+        if (first && first.source.sourceSequence > input.after + 1) {
+            return { ok: false, code: 'source_gap', retainedFrom, highWater, coveredThrough, ...identity(input) };
+        }
+        const events = matched.slice(0, input.limit ?? 100);
+        const nextAfter = events.at(-1)?.source.sourceSequence ?? input.after;
+        return { ok: true, events, retainedFrom, highWater, nextAfter, ...identity(input) };
+    }
+    async allocateSourceEvent(input) {
+        const result = await this.run('allocateEvent', input);
+        const event = result.committedEvents?.[0];
+        if (!event)
+            throw new Error('source_event_missing');
+        return event;
+    }
+    async commitSourceIdentity(event) {
+        await this.run('commitIdentity', { event: pendingSourceEventSchema.parse(event) });
+    }
+    async ingestNative(input) {
+        const result = await this.run('ingest', input);
+        return result.committedEvents ?? [];
+    }
+    async bindTurn(input) {
+        const result = await this.run('bindTurn', input);
+        return result.committedEvents ?? [];
+    }
+    async acknowledgeSource(ack) {
+        await this.run('ack', sourceAckSchema.parse(ack));
+    }
+    async createReplayPin(input) {
+        const result = await this.run('createPin', { identity: input, ttlMs: input.ttlMs });
+        return replayPinResultSchema.parse(result.pin);
+    }
+    async deleteReplayPin(pinId) {
+        await this.run('deletePin', { pinId });
+    }
+    async reclaimSourceEvents(minRetainMs = 0) {
+        await this.run('reclaim', { minRetainMs });
     }
     async putOperation(scopeKey, operation) {
-        this.snapshot.operations[scopeKey] = operationRecordSchema.parse(operation);
-        await this.flush();
-        return this.snapshot.operations[scopeKey];
+        const parsed = operationRecordSchema.parse(operation);
+        await this.run('putOperation', { scopeKey, operation: parsed });
+        return this.snapshot.operations[scopeKey] ?? parsed;
     }
     async putSession(binding) {
-        this.snapshot.sessions[binding.sessionId] = binding;
-        await this.flush();
+        await this.run('upsertRecord', { bucket: 'sessions', key: binding.sessionId, body: binding });
     }
     async putTerminal(record) {
         const parsed = terminalRecordSchema.parse(record);
-        this.snapshot.terminals[parsed.terminalId] = parsed;
-        await this.flush();
+        await this.run('upsertRecord', { bucket: 'terminals', key: parsed.terminalId, body: parsed });
         return parsed;
     }
     async putCheckpoint(checkpoint) {
         const parsed = checkpointRecordSchema.parse(checkpoint);
-        this.snapshot.checkpoints[parsed.checkpointId] = parsed;
-        await this.flush();
+        await this.run('upsertRecord', { bucket: 'checkpoints', key: parsed.checkpointId, body: parsed });
         return parsed;
     }
     async putFileRevision(path, record, workspaceId) {
         if (workspaceId) {
-            (this.snapshot.workspaceFileRevisions[workspaceId] ??= {})[path] = record;
-        }
-        else {
-            this.snapshot.fileRevisions[path] = record;
+            await this.run('upsertRecord', { bucket: `workspace_files:${workspaceId}`, key: path, body: record });
+            return;
         }
-        await this.flush();
+        await this.run('upsertRecord', { bucket: 'file_revisions', key: path, body: record });
     }
     async replaceFileRevisions(records, workspaceId) {
-        if (workspaceId)
-            this.snapshot.workspaceFileRevisions[workspaceId] = { ...records };
-        else
-            this.snapshot.fileRevisions = { ...records };
-        await this.flush();
+        const entries = Object.entries(records).map(([key, body]) => ({ key, body }));
+        await this.run('replaceBucket', { bucket: workspaceId ? `workspace_files:${workspaceId}` : 'file_revisions', entries });
     }
     async markDestroyed() {
-        this.snapshot.destroyed = true;
-        await this.flush();
+        await this.run('setDestroyed', {});
     }
     async replaceSessions(sessions) {
-        this.snapshot.sessions = Object.fromEntries(sessions.map((item) => [item.sessionId, item]));
-        await this.flush();
+        await this.run('replaceBucket', {
+            bucket: 'sessions',
+            entries: sessions.map((session) => ({ key: session.sessionId, body: session })),
+        });
     }
     async setAcceptedLeaseFence(value) {
-        this.snapshot.acceptedLeaseFence = Math.max(this.snapshot.acceptedLeaseFence, value);
-        await this.flush();
+        await this.run('setAcceptedFence', { value });
     }
     async putReceipt(receipt, sourceSequence) {
-        this.snapshot.receipts[sourceEventKey(receipt.source)] =
-            sourceEventReceiptSchema.parse(receipt);
-        this.snapshot.lastSourceSequence = Math.max(this.snapshot.lastSourceSequence, sourceSequence);
-        await this.flush();
+        const parsed = sourceEventReceiptSchema.parse(receipt);
+        await this.run('upsertRecord', { bucket: 'receipts', key: sourceEventKey(parsed.source), body: parsed });
+        void sourceSequence;
     }
     async appendEvent(event) {
         const parsed = pendingSourceEventSchema.parse(event);
-        this.snapshot.events[sourceEventKey(parsed.source)] = parsed;
-        this.snapshot.lastSourceSequence = Math.max(this.snapshot.lastSourceSequence, parsed.source.sourceSequence);
-        await this.flush();
-        return parsed;
-    }
-    async flush() {
-        const snapshot = `${JSON.stringify(this.snapshot)}\n`;
-        const write = this.writes.then(async () => {
-            await mkdir(dirname(this.filePath), { recursive: true, mode: 0o700 });
-            const tempPath = `${this.filePath}.${process.pid}.tmp`;
-            await writeFile(tempPath, snapshot, { mode: 0o600 });
-            await rename(tempPath, this.filePath);
-        });
-        this.writes = write.catch(() => undefined);
-        await write;
-    }
-}
-function sanitizeManagedWrites(value) {
-    if (!value || typeof value !== 'object')
-        return {};
-    const records = {};
-    for (const [key, item] of Object.entries(value)) {
-        if (!item || typeof item !== 'object')
-            continue;
-        const record = item;
-        if (record.writeId !== key || typeof record.workspaceId !== 'string' || typeof record.startedAt !== 'string') {
-            continue;
+        await this.commitSourceIdentity(parsed);
+        return this.snapshot.events[sourceEventKey(parsed.source)] ?? parsed;
+    }
+    async run(type, payload) {
+        if (!this.client)
+            throw new Error('sqlite_not_open');
+        const result = await this.client.request(type, payload);
+        this.apply(result);
+        for (const event of result.committedEvents ?? []) {
+            for (const listener of this.listeners)
+                listener(event);
         }
-        if (record.source !== 'manual' && record.source !== 'agent' && record.source !== 'background')
-            continue;
-        records[key] = {
-            writeId: key,
-            workspaceId: record.workspaceId,
-            source: record.source,
-            startedAt: record.startedAt,
-            pid: integerOrUndefined(record.pid),
-            stoppedAt: typeof record.stoppedAt === 'string' ? record.stoppedAt : undefined,
+        return result;
+    }
+    apply(dump) {
+        const group = dump.snapshot.executionGroup;
+        this.snapshot = {
+            ...emptySnapshot(),
+            ...dump.snapshot,
+            executionGroup: {
+                stopConfirmed: group?.stopConfirmed === true,
+                replaced: group?.replaced === true,
+                pid: group?.pid,
+                startedAt: group?.startedAt,
+                previousPid: group?.previousPid,
+            },
         };
+        this.epochs = dump.epochs ?? [];
+        this.readToken = dump.sourceReadToken ?? this.readToken;
+        this.inboxPending = dump.pendingInbox ?? 0;
+        if (dump.engine)
+            this.engineInfo = dump.engine;
     }
-    return records;
 }
-function approvalKey(sessionId, approvalId) {
-    return `${sessionId}:${approvalId}`;
+async function readLegacy(legacyPath, sqlitePath) {
+    if (!existsSync(legacyPath))
+        return null;
+    const text = await readFile(legacyPath, 'utf8');
+    if (text.startsWith(LEGACY_JOURNAL_SENTINEL.trim())) {
+        if (!existsSync(sqlitePath))
+            throw new Error('runtime_state_missing');
+        return null;
+    }
+    let value;
+    try {
+        value = JSON.parse(text);
+    }
+    catch {
+        throw new Error('legacy_journal_unreadable');
+    }
+    if (!existsSync(`${legacyPath}.bak`))
+        await copyFile(legacyPath, `${legacyPath}.bak`);
+    return { value, readable: true };
 }
-function integerOrUndefined(value) {
-    return typeof value === 'number' && Number.isInteger(value) && value >= 0 ? value : undefined;
+function identity(input) {
+    return { runtimeId: input.runtimeId, generation: input.generation, runtimeEpoch: input.runtimeEpoch };
 }
-function isStoredApproval(value) {
-    if (!value || typeof value !== 'object')
-        return false;
-    const record = value;
-    return (typeof record.approvalId === 'string' &&
-        typeof record.sessionId === 'string' &&
-        typeof record.runtimeEpoch === 'number' &&
-        typeof record.nativeRequestId === 'string' &&
-        (record.status === 'pending' ||
-            record.status === 'approved' ||
-            record.status === 'rejected' ||
-            record.status === 'expired'));
-}
-function highestFence(operations) {
-    return Object.values(operations).reduce((highest, operation) => Math.max(highest, operation.leaseFence ?? 0), 0);
+function emptySnapshot() {
+    return {
+        workspaces: {},
+        workspaceFileRevisions: {},
+        operations: {},
+        receipts: {},
+        events: {},
+        sessions: {},
+        terminals: {},
+        checkpoints: {},
+        fileRevisions: {},
+        approvals: {},
+        managedWrites: {},
+        executionGroup: { stopConfirmed: false, replaced: false },
+        lastSourceSequence: 0,
+        acceptedLeaseFence: 0,
+        destroyed: false,
+    };
 }

package/dist/workspace-runtime/persistence/sqlite-client.js

@@ -0,0 +1,61 @@
+import { existsSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { Worker } from 'node:worker_threads';
+export class SqliteWorkerClient {
+    nextId = 1;
+    pending = new Map();
+    chain = Promise.resolve();
+    worker;
+    constructor(worker) {
+        this.worker = worker;
+        worker.on('message', (message) => {
+            const waiter = this.pending.get(message.id);
+            if (!waiter)
+                return;
+            this.pending.delete(message.id);
+            if (message.ok)
+                waiter.resolve(message.result);
+            else
+                waiter.reject(new Error(message.error || 'sqlite_worker_failed'));
+        });
+        worker.on('error', (error) => {
+            for (const waiter of this.pending.values())
+                waiter.reject(error);
+            this.pending.clear();
+        });
+    }
+    static async open(filePath, legacy) {
+        const client = new SqliteWorkerClient(new Worker(workerUrl(), { execArgv: [], env: { ...process.env, NODE_OPTIONS: '' } }));
+        try {
+            const opened = await client.request('open', { filePath, legacy });
+            return { client, opened };
+        }
+        catch (error) {
+            await client.close().catch(() => undefined);
+            throw error;
+        }
+    }
+    request(type, payload) {
+        const run = this.chain.then(() => this.send(type, payload));
+        this.chain = run.then(() => undefined, () => undefined);
+        return run;
+    }
+    async close() {
+        await this.request('close', {}).catch(() => undefined);
+        await this.worker.terminate();
+    }
+    send(type, payload) {
+        const id = this.nextId;
+        this.nextId += 1;
+        return new Promise((resolve, reject) => {
+            this.pending.set(id, { resolve: (value) => resolve(value), reject });
+            this.worker.postMessage({ id, type, payload });
+        });
+    }
+}
+function workerUrl() {
+    const compiled = new URL('./sqlite-worker.js', import.meta.url);
+    if (existsSync(fileURLToPath(compiled)))
+        return compiled;
+    return new URL('./sqlite-worker.ts', import.meta.url);
+}