amalgm 0.1.155 → 0.1.156
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/shared-realtime-tunnel.js +518 -0
- package/lib/tunnel-events.js +27 -0
- package/package.json +1 -1
- package/runtime/scripts/amalgm-mcp/server/routes/state.js +75 -0
- package/runtime/scripts/amalgm-mcp/state/attachments.js +187 -0
- package/runtime/scripts/amalgm-mcp/state/db.js +144 -0
- package/runtime/scripts/amalgm-mcp/state/docs.js +230 -24
- package/runtime/scripts/amalgm-mcp/state/events.js +20 -2
- package/runtime/scripts/amalgm-mcp/state/mutations.js +800 -0
- package/runtime/scripts/amalgm-mcp/state/promotions.js +271 -0
- package/runtime/scripts/amalgm-mcp/state/replicas.js +200 -0
- package/runtime/scripts/amalgm-mcp/state/rest.js +4 -0
- package/runtime/scripts/amalgm-mcp/state/shared-rest.js +267 -0
- package/runtime/scripts/amalgm-mcp/tests/state-mutations.test.js +337 -0
- package/runtime/scripts/amalgm-mcp/tests/state-shared-replica.test.js +283 -0
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const { cachedStatement, openLocalDb } = require('./db');
|
|
4
|
+
const { appendStateEvent } = require('./events');
|
|
5
|
+
|
|
6
|
+
const SHARED_RESOURCE_ID = /^shr_[A-Za-z0-9_-]{16,128}$/;
|
|
7
|
+
const ATTACHMENT_ID = /^[A-Za-z0-9_-]{16,160}$/;
|
|
8
|
+
|
|
9
|
+
function invalid(message, statusCode = 400) {
|
|
10
|
+
return Object.assign(new Error(message), { statusCode });
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
function requiredText(value, name, max = 4096) {
|
|
14
|
+
const text = typeof value === 'string' ? value.trim() : '';
|
|
15
|
+
if (!text || text.length > max || /[\u0000-\u001f\u007f]/.test(text)) {
|
|
16
|
+
throw invalid(`${name} must be a non-empty string of at most ${max} characters`);
|
|
17
|
+
}
|
|
18
|
+
return text;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function normalizeRow(row) {
|
|
22
|
+
if (!row) return null;
|
|
23
|
+
return {
|
|
24
|
+
attachmentId: row.attachment_id,
|
|
25
|
+
resourceId: row.resource_id,
|
|
26
|
+
localPath: row.local_path,
|
|
27
|
+
state: row.attachment_state,
|
|
28
|
+
attemptCount: Number(row.attempt_count),
|
|
29
|
+
nextAttemptAt: row.next_attempt_at || null,
|
|
30
|
+
lastError: row.last_error || null,
|
|
31
|
+
createdAt: row.created_at,
|
|
32
|
+
updatedAt: row.updated_at,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function enqueueAttachment(input, options = {}) {
|
|
37
|
+
const attachmentId = requiredText(input?.attachmentId, 'attachmentId', 160);
|
|
38
|
+
const resourceId = requiredText(input?.resourceId, 'resourceId', 132);
|
|
39
|
+
const localPath = requiredText(input?.localPath, 'localPath');
|
|
40
|
+
if (!ATTACHMENT_ID.test(attachmentId)) throw invalid('attachmentId has an invalid format');
|
|
41
|
+
if (!SHARED_RESOURCE_ID.test(resourceId)) throw invalid('resourceId has an invalid format');
|
|
42
|
+
const database = options.database || openLocalDb();
|
|
43
|
+
const existing = cachedStatement(
|
|
44
|
+
database,
|
|
45
|
+
'SELECT * FROM shared_resource_attachments WHERE attachment_id = ?',
|
|
46
|
+
).get(attachmentId);
|
|
47
|
+
if (existing) {
|
|
48
|
+
const attachment = normalizeRow(existing);
|
|
49
|
+
if (attachment.resourceId !== resourceId || attachment.localPath !== localPath) {
|
|
50
|
+
throw invalid(`attachmentId ${attachmentId} was reused with different content`, 409);
|
|
51
|
+
}
|
|
52
|
+
return { attachment, duplicate: true };
|
|
53
|
+
}
|
|
54
|
+
const now = new Date().toISOString();
|
|
55
|
+
try {
|
|
56
|
+
cachedStatement(database, `
|
|
57
|
+
INSERT INTO shared_resource_attachments (
|
|
58
|
+
attachment_id, resource_id, local_path, created_at, updated_at
|
|
59
|
+
) VALUES (?, ?, ?, ?, ?)
|
|
60
|
+
`).run(attachmentId, resourceId, localPath, now, now);
|
|
61
|
+
} catch (error) {
|
|
62
|
+
if (String(error?.code || '').includes('CONSTRAINT')) {
|
|
63
|
+
throw invalid('that resource or local path already has an attachment request', 409);
|
|
64
|
+
}
|
|
65
|
+
throw error;
|
|
66
|
+
}
|
|
67
|
+
const attachment = normalizeRow(
|
|
68
|
+
cachedStatement(database, 'SELECT * FROM shared_resource_attachments WHERE attachment_id = ?')
|
|
69
|
+
.get(attachmentId),
|
|
70
|
+
);
|
|
71
|
+
appendStateEvent({
|
|
72
|
+
resource: 'shared:attachments',
|
|
73
|
+
op: 'update',
|
|
74
|
+
id: attachmentId,
|
|
75
|
+
sharedResourceId: resourceId,
|
|
76
|
+
value: { state: 'pending' },
|
|
77
|
+
source: 'shared:attachment',
|
|
78
|
+
});
|
|
79
|
+
return { attachment, duplicate: false };
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function claimAttachments(options = {}) {
|
|
83
|
+
const database = options.database || openLocalDb();
|
|
84
|
+
const limit = Math.max(1, Math.min(Number(options.limit) || 4, 20));
|
|
85
|
+
const leaseMs = Math.max(1000, Number(options.leaseMs) || 30_000);
|
|
86
|
+
const now = new Date().toISOString();
|
|
87
|
+
const leaseUntil = new Date(Date.now() + leaseMs).toISOString();
|
|
88
|
+
return database.transaction(() => {
|
|
89
|
+
const rows = cachedStatement(database, `
|
|
90
|
+
SELECT * FROM shared_resource_attachments
|
|
91
|
+
WHERE attachment_state = 'pending'
|
|
92
|
+
OR (attachment_state = 'syncing' AND (next_attempt_at IS NULL OR next_attempt_at <= ?))
|
|
93
|
+
ORDER BY created_at ASC
|
|
94
|
+
LIMIT ?
|
|
95
|
+
`).all(now, limit);
|
|
96
|
+
const claim = cachedStatement(database, `
|
|
97
|
+
UPDATE shared_resource_attachments
|
|
98
|
+
SET attachment_state = 'syncing',
|
|
99
|
+
attempt_count = attempt_count + 1,
|
|
100
|
+
next_attempt_at = ?,
|
|
101
|
+
updated_at = ?
|
|
102
|
+
WHERE attachment_id = ?
|
|
103
|
+
`);
|
|
104
|
+
return rows.map((row) => {
|
|
105
|
+
claim.run(leaseUntil, now, row.attachment_id);
|
|
106
|
+
return normalizeRow(
|
|
107
|
+
cachedStatement(database, 'SELECT * FROM shared_resource_attachments WHERE attachment_id = ?')
|
|
108
|
+
.get(row.attachment_id),
|
|
109
|
+
);
|
|
110
|
+
});
|
|
111
|
+
})();
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function acknowledgeAttachment(attachmentIdInput, payload, options = {}) {
|
|
115
|
+
const attachmentId = requiredText(attachmentIdInput, 'attachmentId', 160);
|
|
116
|
+
const database = options.database || openLocalDb();
|
|
117
|
+
const row = cachedStatement(
|
|
118
|
+
database,
|
|
119
|
+
'SELECT * FROM shared_resource_attachments WHERE attachment_id = ?',
|
|
120
|
+
).get(attachmentId);
|
|
121
|
+
if (!row) throw invalid(`unknown attachment ${attachmentId}`, 404);
|
|
122
|
+
const attachment = normalizeRow(row);
|
|
123
|
+
if (attachment.state === 'attached') return { attachment, duplicate: true };
|
|
124
|
+
if (payload?.resourceId !== attachment.resourceId) {
|
|
125
|
+
throw invalid('attachment response resource mismatch', 409);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const installed = require('./docs').installCloudDocSnapshot(attachment.localPath, payload);
|
|
129
|
+
const now = new Date().toISOString();
|
|
130
|
+
cachedStatement(database, `
|
|
131
|
+
UPDATE shared_resource_attachments
|
|
132
|
+
SET attachment_state = 'attached',
|
|
133
|
+
next_attempt_at = NULL,
|
|
134
|
+
last_error = NULL,
|
|
135
|
+
updated_at = ?
|
|
136
|
+
WHERE attachment_id = ?
|
|
137
|
+
`).run(now, attachmentId);
|
|
138
|
+
return {
|
|
139
|
+
attachment: normalizeRow(
|
|
140
|
+
cachedStatement(database, 'SELECT * FROM shared_resource_attachments WHERE attachment_id = ?')
|
|
141
|
+
.get(attachmentId),
|
|
142
|
+
),
|
|
143
|
+
replica: installed.replica,
|
|
144
|
+
duplicate: false,
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function failAttachment(attachmentIdInput, failure, options = {}) {
|
|
149
|
+
const attachmentId = requiredText(attachmentIdInput, 'attachmentId', 160);
|
|
150
|
+
const retryable = failure?.retryable !== false;
|
|
151
|
+
const database = options.database || openLocalDb();
|
|
152
|
+
const result = cachedStatement(database, `
|
|
153
|
+
UPDATE shared_resource_attachments
|
|
154
|
+
SET attachment_state = ?,
|
|
155
|
+
next_attempt_at = ?,
|
|
156
|
+
last_error = ?,
|
|
157
|
+
updated_at = ?
|
|
158
|
+
WHERE attachment_id = ? AND attachment_state != 'attached'
|
|
159
|
+
`).run(
|
|
160
|
+
retryable ? 'syncing' : 'rejected',
|
|
161
|
+
retryable ? new Date(Date.now() + 1000).toISOString() : null,
|
|
162
|
+
String(failure?.message || 'attachment failed').slice(0, 4000),
|
|
163
|
+
new Date().toISOString(),
|
|
164
|
+
attachmentId,
|
|
165
|
+
);
|
|
166
|
+
if (!result.changes) throw invalid(`unknown or completed attachment ${attachmentId}`, 404);
|
|
167
|
+
return normalizeRow(
|
|
168
|
+
cachedStatement(database, 'SELECT * FROM shared_resource_attachments WHERE attachment_id = ?')
|
|
169
|
+
.get(attachmentId),
|
|
170
|
+
);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function getAttachment(attachmentId, options = {}) {
|
|
174
|
+
const database = options.database || openLocalDb();
|
|
175
|
+
return normalizeRow(
|
|
176
|
+
cachedStatement(database, 'SELECT * FROM shared_resource_attachments WHERE attachment_id = ?')
|
|
177
|
+
.get(attachmentId),
|
|
178
|
+
);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
module.exports = {
|
|
182
|
+
acknowledgeAttachment,
|
|
183
|
+
claimAttachments,
|
|
184
|
+
enqueueAttachment,
|
|
185
|
+
failAttachment,
|
|
186
|
+
getAttachment,
|
|
187
|
+
};
|
|
@@ -68,6 +68,8 @@ function migrate(database = openLocalDb()) {
|
|
|
68
68
|
value_json TEXT,
|
|
69
69
|
patch_json TEXT,
|
|
70
70
|
client_mutation_id TEXT,
|
|
71
|
+
mutation_id TEXT,
|
|
72
|
+
shared_resource_id TEXT,
|
|
71
73
|
source TEXT,
|
|
72
74
|
resource_version INTEGER
|
|
73
75
|
);
|
|
@@ -77,6 +79,136 @@ function migrate(database = openLocalDb()) {
|
|
|
77
79
|
-- from earlier releases.
|
|
78
80
|
DROP INDEX IF EXISTS event_log_resource_seq_idx;
|
|
79
81
|
|
|
82
|
+
-- Durable edit identity and cloud delivery state live here. event_log
|
|
83
|
+
-- remains the short published/replay projection: a mutation enters it
|
|
84
|
+
-- only after the materialized local store is current.
|
|
85
|
+
CREATE TABLE IF NOT EXISTS mutation_journal (
|
|
86
|
+
journal_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
87
|
+
channel_id TEXT NOT NULL,
|
|
88
|
+
local_resource TEXT NOT NULL,
|
|
89
|
+
shared_resource_id TEXT,
|
|
90
|
+
mutation_id TEXT NOT NULL,
|
|
91
|
+
device_id TEXT,
|
|
92
|
+
authority_epoch INTEGER NOT NULL DEFAULT 0 CHECK (authority_epoch >= 0),
|
|
93
|
+
base_cloud_version INTEGER NOT NULL DEFAULT 0 CHECK (base_cloud_version >= 0),
|
|
94
|
+
official_cloud_version INTEGER CHECK (official_cloud_version IS NULL OR official_cloud_version >= 0),
|
|
95
|
+
contract TEXT NOT NULL,
|
|
96
|
+
schema_version INTEGER NOT NULL CHECK (schema_version > 0),
|
|
97
|
+
operation_kind TEXT NOT NULL,
|
|
98
|
+
operation_json TEXT NOT NULL,
|
|
99
|
+
mutation_origin TEXT NOT NULL DEFAULT 'local'
|
|
100
|
+
CHECK (mutation_origin IN ('local', 'cloud')),
|
|
101
|
+
mutation_state TEXT NOT NULL DEFAULT 'saving-local'
|
|
102
|
+
CHECK (mutation_state IN (
|
|
103
|
+
'saving-local',
|
|
104
|
+
'saved-local',
|
|
105
|
+
'syncing',
|
|
106
|
+
'synced',
|
|
107
|
+
'rejected',
|
|
108
|
+
'conflicted'
|
|
109
|
+
)),
|
|
110
|
+
local_revision INTEGER,
|
|
111
|
+
materialized_at TEXT,
|
|
112
|
+
event_seq INTEGER,
|
|
113
|
+
attempt_count INTEGER NOT NULL DEFAULT 0 CHECK (attempt_count >= 0),
|
|
114
|
+
next_attempt_at TEXT,
|
|
115
|
+
last_error TEXT,
|
|
116
|
+
cloud_committed_at TEXT,
|
|
117
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
|
|
118
|
+
updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
|
|
119
|
+
UNIQUE (channel_id, mutation_id)
|
|
120
|
+
);
|
|
121
|
+
|
|
122
|
+
CREATE INDEX IF NOT EXISTS mutation_journal_cloud_outbox_idx
|
|
123
|
+
ON mutation_journal (mutation_state, next_attempt_at, journal_id)
|
|
124
|
+
WHERE shared_resource_id IS NOT NULL AND mutation_origin = 'local';
|
|
125
|
+
|
|
126
|
+
CREATE INDEX IF NOT EXISTS mutation_journal_shared_version_idx
|
|
127
|
+
ON mutation_journal (shared_resource_id, official_cloud_version)
|
|
128
|
+
WHERE shared_resource_id IS NOT NULL;
|
|
129
|
+
|
|
130
|
+
CREATE UNIQUE INDEX IF NOT EXISTS mutation_journal_official_version_unique_idx
|
|
131
|
+
ON mutation_journal (shared_resource_id, official_cloud_version)
|
|
132
|
+
WHERE shared_resource_id IS NOT NULL AND official_cloud_version IS NOT NULL;
|
|
133
|
+
|
|
134
|
+
CREATE INDEX IF NOT EXISTS mutation_journal_event_seq_idx
|
|
135
|
+
ON mutation_journal (event_seq)
|
|
136
|
+
WHERE event_seq IS NOT NULL;
|
|
137
|
+
|
|
138
|
+
-- Runtime-owned binding between one cloud-authoritative resource and its
|
|
139
|
+
-- real local materialization. Browsers cannot turn a local file into a
|
|
140
|
+
-- cloud source by adding a request flag; promotion/join installs this
|
|
141
|
+
-- binding, and every edit thereafter discovers authority here.
|
|
142
|
+
CREATE TABLE IF NOT EXISTS shared_resource_replicas (
|
|
143
|
+
resource_id TEXT PRIMARY KEY,
|
|
144
|
+
local_resource TEXT NOT NULL UNIQUE,
|
|
145
|
+
local_path TEXT NOT NULL UNIQUE,
|
|
146
|
+
contract TEXT NOT NULL,
|
|
147
|
+
schema_version INTEGER NOT NULL CHECK (schema_version > 0),
|
|
148
|
+
authority_epoch INTEGER NOT NULL CHECK (authority_epoch > 0),
|
|
149
|
+
applied_version INTEGER NOT NULL DEFAULT 0 CHECK (applied_version >= 0),
|
|
150
|
+
snapshot_version INTEGER NOT NULL DEFAULT 0 CHECK (snapshot_version >= 0),
|
|
151
|
+
snapshot_checksum TEXT,
|
|
152
|
+
sync_status TEXT NOT NULL DEFAULT 'synced'
|
|
153
|
+
CHECK (sync_status IN (
|
|
154
|
+
'saving-local',
|
|
155
|
+
'saved-local',
|
|
156
|
+
'syncing',
|
|
157
|
+
'synced',
|
|
158
|
+
'rejected',
|
|
159
|
+
'conflicted'
|
|
160
|
+
)),
|
|
161
|
+
last_error TEXT,
|
|
162
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
|
|
163
|
+
updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
|
|
164
|
+
CHECK (snapshot_version <= applied_version)
|
|
165
|
+
);
|
|
166
|
+
|
|
167
|
+
CREATE INDEX IF NOT EXISTS shared_resource_replicas_path_idx
|
|
168
|
+
ON shared_resource_replicas (local_path);
|
|
169
|
+
|
|
170
|
+
CREATE TABLE IF NOT EXISTS shared_resource_promotions (
|
|
171
|
+
promotion_id TEXT PRIMARY KEY,
|
|
172
|
+
resource_id TEXT NOT NULL UNIQUE,
|
|
173
|
+
local_resource TEXT NOT NULL UNIQUE,
|
|
174
|
+
local_path TEXT NOT NULL UNIQUE,
|
|
175
|
+
contract TEXT NOT NULL,
|
|
176
|
+
schema_version INTEGER NOT NULL CHECK (schema_version > 0),
|
|
177
|
+
snapshot_b64 TEXT NOT NULL,
|
|
178
|
+
snapshot_checksum TEXT NOT NULL,
|
|
179
|
+
snapshot_bytes INTEGER NOT NULL CHECK (snapshot_bytes >= 0),
|
|
180
|
+
content_bytes INTEGER CHECK (content_bytes IS NULL OR content_bytes >= 0),
|
|
181
|
+
name TEXT,
|
|
182
|
+
media_type TEXT,
|
|
183
|
+
promotion_state TEXT NOT NULL DEFAULT 'pending'
|
|
184
|
+
CHECK (promotion_state IN ('pending', 'syncing', 'promoted', 'rejected')),
|
|
185
|
+
attempt_count INTEGER NOT NULL DEFAULT 0 CHECK (attempt_count >= 0),
|
|
186
|
+
next_attempt_at TEXT,
|
|
187
|
+
last_error TEXT,
|
|
188
|
+
snapshot_key TEXT,
|
|
189
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
|
|
190
|
+
updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
|
|
191
|
+
);
|
|
192
|
+
|
|
193
|
+
CREATE INDEX IF NOT EXISTS shared_resource_promotions_outbox_idx
|
|
194
|
+
ON shared_resource_promotions (promotion_state, next_attempt_at, created_at);
|
|
195
|
+
|
|
196
|
+
CREATE TABLE IF NOT EXISTS shared_resource_attachments (
|
|
197
|
+
attachment_id TEXT PRIMARY KEY,
|
|
198
|
+
resource_id TEXT NOT NULL UNIQUE,
|
|
199
|
+
local_path TEXT NOT NULL UNIQUE,
|
|
200
|
+
attachment_state TEXT NOT NULL DEFAULT 'pending'
|
|
201
|
+
CHECK (attachment_state IN ('pending', 'syncing', 'attached', 'rejected')),
|
|
202
|
+
attempt_count INTEGER NOT NULL DEFAULT 0 CHECK (attempt_count >= 0),
|
|
203
|
+
next_attempt_at TEXT,
|
|
204
|
+
last_error TEXT,
|
|
205
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
|
|
206
|
+
updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
|
|
207
|
+
);
|
|
208
|
+
|
|
209
|
+
CREATE INDEX IF NOT EXISTS shared_resource_attachments_outbox_idx
|
|
210
|
+
ON shared_resource_attachments (attachment_state, next_attempt_at, created_at);
|
|
211
|
+
|
|
80
212
|
CREATE TABLE IF NOT EXISTS local_meta (
|
|
81
213
|
key TEXT PRIMARY KEY,
|
|
82
214
|
value TEXT,
|
|
@@ -466,9 +598,21 @@ function migrate(database = openLocalDb()) {
|
|
|
466
598
|
ON browser_login_sessions(expires_at);
|
|
467
599
|
|
|
468
600
|
`);
|
|
601
|
+
|
|
602
|
+
// event_log is already present on every installed machine. These nullable
|
|
603
|
+
// projection fields are additive and leave every legacy row/wire shape
|
|
604
|
+
// untouched while new SDK mutations carry their stable identity.
|
|
605
|
+
ensureColumn(database, 'event_log', 'mutation_id', 'TEXT');
|
|
606
|
+
ensureColumn(database, 'event_log', 'shared_resource_id', 'TEXT');
|
|
469
607
|
require('./migrations').runLegacyMigrations(database);
|
|
470
608
|
}
|
|
471
609
|
|
|
610
|
+
function ensureColumn(database, tableName, columnName, declaration) {
|
|
611
|
+
const columns = database.prepare(`PRAGMA table_info(${tableName})`).all();
|
|
612
|
+
if (columns.some((column) => column.name === columnName)) return;
|
|
613
|
+
database.exec(`ALTER TABLE ${tableName} ADD COLUMN ${columnName} ${declaration}`);
|
|
614
|
+
}
|
|
615
|
+
|
|
472
616
|
// ---------------------------------------------------------------------------
|
|
473
617
|
// Lazily created tables. The DDL lives here so this file owns every schema
|
|
474
618
|
// statement, but these tables are only created when their feature is first
|
|
@@ -116,6 +116,10 @@ function hashText(text) {
|
|
|
116
116
|
return crypto.createHash('sha256').update(text, 'utf8').digest('hex');
|
|
117
117
|
}
|
|
118
118
|
|
|
119
|
+
function newMutationId() {
|
|
120
|
+
return `mut_${crypto.randomBytes(16).toString('base64url')}`;
|
|
121
|
+
}
|
|
122
|
+
|
|
119
123
|
// The only way lastDiskText is ever assigned: the hash is what persistence
|
|
120
124
|
// writes, and hashing at assignment (rare: open, reconcile, flush) keeps the
|
|
121
125
|
// full-text SHA-256 off the per-batch persist path. `origin` says how the
|
|
@@ -496,6 +500,42 @@ function closeDocEntry(Y, entry) {
|
|
|
496
500
|
openDocs.delete(entry.path);
|
|
497
501
|
}
|
|
498
502
|
|
|
503
|
+
function recoverPendingDocMutations(Y, entry) {
|
|
504
|
+
const pending = require('./mutations').listMutations({ state: 'saving-local', limit: 1000 })
|
|
505
|
+
.filter((mutation) => mutation.localResource === entry.resource);
|
|
506
|
+
for (const mutation of pending) {
|
|
507
|
+
if (
|
|
508
|
+
mutation.operationKind !== 'yjs.update'
|
|
509
|
+
|| typeof mutation.operation?.update !== 'string'
|
|
510
|
+
) {
|
|
511
|
+
continue;
|
|
512
|
+
}
|
|
513
|
+
const bytes = Buffer.from(mutation.operation.update, 'base64');
|
|
514
|
+
try {
|
|
515
|
+
Y.decodeUpdate(new Uint8Array(bytes));
|
|
516
|
+
} catch (error) {
|
|
517
|
+
throw invalid(`cannot recover mutation ${mutation.mutationId}: ${error?.message || error}`, 500);
|
|
518
|
+
}
|
|
519
|
+
const origin = { amalgmMutation: true, source: 'recovery', patch: null };
|
|
520
|
+
Y.applyUpdate(entry.doc, new Uint8Array(bytes), origin);
|
|
521
|
+
persistDocState(Y, entry);
|
|
522
|
+
if (mutation.sharedResourceId) flushDocToDisk(Y, entry);
|
|
523
|
+
else scheduleDiskWrite(Y, entry);
|
|
524
|
+
require('./mutations').completeMaterializedMutation(
|
|
525
|
+
mutation.channelId,
|
|
526
|
+
mutation.mutationId,
|
|
527
|
+
{
|
|
528
|
+
resource: entry.resource,
|
|
529
|
+
op: 'update',
|
|
530
|
+
id: entry.path,
|
|
531
|
+
patch: origin.patch || { yjs: mutation.operation.update },
|
|
532
|
+
source: mutation.origin === 'cloud' ? 'doc:cloud-recovery' : 'doc:recovery',
|
|
533
|
+
resourceVersion: mutation.officialCloudVersion,
|
|
534
|
+
},
|
|
535
|
+
);
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
|
|
499
539
|
// Called before inserting a new entry: evicts down to MAX_OPEN_DOCS - 1 so
|
|
500
540
|
// the insert lands exactly at the cap, never past it.
|
|
501
541
|
function evictIfNeeded(Y) {
|
|
@@ -588,6 +628,14 @@ function loadDocEntry(pathInput) {
|
|
|
588
628
|
patch.conflict = entry.pendingConflict;
|
|
589
629
|
entry.pendingConflict = null;
|
|
590
630
|
}
|
|
631
|
+
// SDK mutations own their journal row and event projection explicitly:
|
|
632
|
+
// durable journal first, then Y.Doc + file materialization, then event.
|
|
633
|
+
// The observer still captures the exact emitted update, but must not
|
|
634
|
+
// append an early event or schedule the shared file write after ack.
|
|
635
|
+
if (origin && typeof origin === 'object' && origin.amalgmMutation === true) {
|
|
636
|
+
origin.patch = patch;
|
|
637
|
+
return;
|
|
638
|
+
}
|
|
591
639
|
appendStateEvent({
|
|
592
640
|
resource: entry.resource,
|
|
593
641
|
op: 'update',
|
|
@@ -603,6 +651,18 @@ function loadDocEntry(pathInput) {
|
|
|
603
651
|
// write debounce) gets the doc written back out — never spliced over it.
|
|
604
652
|
seedFromDisk(Y, entry, persisted);
|
|
605
653
|
|
|
654
|
+
// A process can die after the durable journal insert but before Y.Doc/file
|
|
655
|
+
// materialization or event projection. Replay those exact operation IDs
|
|
656
|
+
// before this open is acknowledged, so a restart cannot bury an edit.
|
|
657
|
+
try {
|
|
658
|
+
recoverPendingDocMutations(Y, entry);
|
|
659
|
+
} catch (error) {
|
|
660
|
+
if (entry.diskWriteTimer) clearTimeout(entry.diskWriteTimer);
|
|
661
|
+
if (entry.reconcileTimer) clearTimeout(entry.reconcileTimer);
|
|
662
|
+
doc.destroy();
|
|
663
|
+
throw error;
|
|
664
|
+
}
|
|
665
|
+
|
|
606
666
|
evictIfNeeded(Y);
|
|
607
667
|
openDocs.set(resolved, entry);
|
|
608
668
|
watchDocFile(Y, entry);
|
|
@@ -761,40 +821,183 @@ function writeDocText(pathInput, textInput, options = {}) {
|
|
|
761
821
|
return respond(merge, nextText, conflict);
|
|
762
822
|
}
|
|
763
823
|
|
|
764
|
-
function
|
|
765
|
-
const
|
|
766
|
-
|
|
767
|
-
if (inputs.length === 0) return { applied: 0, seq: currentSeq() };
|
|
824
|
+
function normalizeDocMutations(Y, mutationsInput) {
|
|
825
|
+
const inputs = Array.isArray(mutationsInput) ? mutationsInput : [mutationsInput];
|
|
826
|
+
if (inputs.length === 0) return [];
|
|
768
827
|
if (inputs.length > MAX_UPDATES_PER_CALL) {
|
|
769
828
|
throw invalid(`at most ${MAX_UPDATES_PER_CALL} updates per call (got ${inputs.length})`);
|
|
770
829
|
}
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
830
|
+
return inputs.map((input, index) => {
|
|
831
|
+
const mutationId = typeof input?.mutationId === 'string' && input.mutationId
|
|
832
|
+
? input.mutationId
|
|
833
|
+
: newMutationId();
|
|
834
|
+
const update = input?.update;
|
|
835
|
+
if (typeof update !== 'string' || !update) {
|
|
836
|
+
throw invalid(`mutations[${index}].update must be a base64-encoded Yjs update`);
|
|
837
|
+
}
|
|
838
|
+
const bytes = Buffer.from(update, 'base64');
|
|
839
|
+
try {
|
|
840
|
+
// Decode before journaling so malformed client data remains a request
|
|
841
|
+
// error, not durable recovery work.
|
|
842
|
+
Y.decodeUpdate(new Uint8Array(bytes));
|
|
843
|
+
} catch (error) {
|
|
844
|
+
throw invalid(`invalid Yjs update: ${error?.message || error}`);
|
|
774
845
|
}
|
|
775
|
-
return
|
|
846
|
+
return { mutationId, update, bytes };
|
|
776
847
|
});
|
|
848
|
+
}
|
|
777
849
|
|
|
850
|
+
function applyDocMutations(pathInput, mutationsInput) {
|
|
851
|
+
const Y = loadYjs();
|
|
852
|
+
const mutations = normalizeDocMutations(Y, mutationsInput);
|
|
853
|
+
if (mutations.length === 0) return { applied: 0, seq: currentSeq(), mutationIds: [] };
|
|
778
854
|
const entry = loadDocEntry(pathInput);
|
|
779
855
|
entry.lastAccess = Date.now();
|
|
780
|
-
|
|
856
|
+
const results = [];
|
|
857
|
+
|
|
858
|
+
for (const mutation of mutations) {
|
|
859
|
+
const replica = require('./replicas').getReplicaByLocalResource(entry.resource);
|
|
860
|
+
const operation = { kind: 'yjs.update', update: mutation.update };
|
|
861
|
+
const origin = { amalgmMutation: true, source: 'local', patch: null };
|
|
862
|
+
const result = require('./mutations').commitLocalMutation({
|
|
863
|
+
channelId: replica ? `resource:${replica.resourceId}` : entry.resource,
|
|
864
|
+
localResource: entry.resource,
|
|
865
|
+
sharedResourceId: replica?.resourceId,
|
|
866
|
+
mutationId: mutation.mutationId,
|
|
867
|
+
deviceId: replica ? process.env.AMALGM_DEVICE_ID : null,
|
|
868
|
+
authorityEpoch: replica?.authorityEpoch || 0,
|
|
869
|
+
baseCloudVersion: replica?.appliedVersion || 0,
|
|
870
|
+
contract: replica?.contract || 'text-yjs@1',
|
|
871
|
+
schemaVersion: replica?.schemaVersion || 1,
|
|
872
|
+
operationKind: 'yjs.update',
|
|
873
|
+
operation,
|
|
874
|
+
}, {
|
|
875
|
+
materialize() {
|
|
876
|
+
Y.applyUpdate(entry.doc, new Uint8Array(mutation.bytes), origin);
|
|
877
|
+
persistDocState(Y, entry);
|
|
878
|
+
if (replica) {
|
|
879
|
+
// Shared files have a strict saved-local boundary: the user-visible
|
|
880
|
+
// file is current before the event/HTTP acknowledgement.
|
|
881
|
+
flushDocToDisk(Y, entry);
|
|
882
|
+
} else {
|
|
883
|
+
// Preserve the existing local-only write-back behavior.
|
|
884
|
+
scheduleDiskWrite(Y, entry);
|
|
885
|
+
}
|
|
886
|
+
},
|
|
887
|
+
event: () => ({
|
|
888
|
+
resource: entry.resource,
|
|
889
|
+
op: 'update',
|
|
890
|
+
id: entry.path,
|
|
891
|
+
patch: origin.patch || { yjs: mutation.update },
|
|
892
|
+
source: 'doc:update',
|
|
893
|
+
}),
|
|
894
|
+
});
|
|
895
|
+
results.push(result);
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
return {
|
|
899
|
+
applied: mutations.length,
|
|
900
|
+
seq: currentSeq(),
|
|
901
|
+
mutationIds: mutations.map((mutation) => mutation.mutationId),
|
|
902
|
+
states: results.map((result) => result.mutation.state),
|
|
903
|
+
};
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
function applyDocUpdates(pathInput, updatesInput) {
|
|
907
|
+
const inputs = Array.isArray(updatesInput) ? updatesInput : [updatesInput];
|
|
908
|
+
return applyDocMutations(
|
|
909
|
+
pathInput,
|
|
910
|
+
inputs.map((update) => ({ mutationId: newMutationId(), update })),
|
|
911
|
+
);
|
|
912
|
+
}
|
|
913
|
+
|
|
914
|
+
function applyCloudDocMutation(input) {
|
|
915
|
+
const envelope = input?.envelope || {};
|
|
916
|
+
if (envelope.operationKind !== 'yjs.update') {
|
|
917
|
+
throw invalid(`unsupported text operation ${JSON.stringify(envelope.operationKind)}`, 409);
|
|
918
|
+
}
|
|
919
|
+
const replica = require('./replicas').getReplicaByResourceId(envelope.resourceId);
|
|
920
|
+
if (!replica) throw invalid(`shared resource ${envelope.resourceId} has no local file`, 404);
|
|
921
|
+
const Y = loadYjs();
|
|
922
|
+
const [mutation] = normalizeDocMutations(Y, [{
|
|
923
|
+
mutationId: envelope.mutationId,
|
|
924
|
+
update: envelope.operation?.update,
|
|
925
|
+
}]);
|
|
926
|
+
const entry = loadDocEntry(replica.localPath);
|
|
927
|
+
if (entry.resource !== replica.localResource) {
|
|
928
|
+
throw invalid(`shared resource ${envelope.resourceId} local binding changed`, 409);
|
|
929
|
+
}
|
|
930
|
+
const origin = { amalgmMutation: true, source: 'cloud', patch: null };
|
|
931
|
+
return require('./mutations').ingestCloudMutation({
|
|
932
|
+
envelope,
|
|
933
|
+
version: input?.version,
|
|
934
|
+
committedAt: input?.committedAt,
|
|
935
|
+
localResource: entry.resource,
|
|
936
|
+
}, {
|
|
937
|
+
materialize() {
|
|
938
|
+
Y.applyUpdate(entry.doc, new Uint8Array(mutation.bytes), origin);
|
|
939
|
+
persistDocState(Y, entry);
|
|
940
|
+
// Recipient file first; the projected event that wakes its UI follows.
|
|
941
|
+
flushDocToDisk(Y, entry);
|
|
942
|
+
},
|
|
943
|
+
event: () => ({
|
|
944
|
+
resource: entry.resource,
|
|
945
|
+
op: 'update',
|
|
946
|
+
id: entry.path,
|
|
947
|
+
patch: origin.patch || { yjs: mutation.update },
|
|
948
|
+
source: 'doc:cloud',
|
|
949
|
+
resourceVersion: Number(input.version),
|
|
950
|
+
}),
|
|
951
|
+
});
|
|
952
|
+
}
|
|
953
|
+
|
|
954
|
+
function installCloudDocSnapshot(pathInput, input) {
|
|
955
|
+
const Y = loadYjs();
|
|
956
|
+
const snapshotBase64 = input?.snapshotBase64;
|
|
957
|
+
if (typeof snapshotBase64 !== 'string' || !snapshotBase64) {
|
|
958
|
+
throw invalid('snapshotBase64 is required');
|
|
959
|
+
}
|
|
960
|
+
const bytes = Buffer.from(snapshotBase64, 'base64');
|
|
961
|
+
const checksum = crypto.createHash('sha256').update(bytes).digest('hex');
|
|
962
|
+
if (checksum !== input?.snapshotChecksum) throw invalid('snapshot checksum mismatch', 409);
|
|
781
963
|
try {
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
} finally {
|
|
791
|
-
// One persist per batch, before the HTTP response acknowledges: that is
|
|
792
|
-
// the whole durability contract. It also runs when a later update in the
|
|
793
|
-
// batch is rejected — events for the applied prefix are already in the
|
|
794
|
-
// log, and the persisted state must never lag them.
|
|
795
|
-
if (applied > 0) persistDocState(Y, entry);
|
|
964
|
+
Y.decodeUpdate(new Uint8Array(bytes));
|
|
965
|
+
} catch (error) {
|
|
966
|
+
throw invalid(`invalid Yjs snapshot: ${error?.message || error}`);
|
|
967
|
+
}
|
|
968
|
+
const entry = loadDocEntry(pathInput);
|
|
969
|
+
const currentText = entry.doc.getText(TEXT_KEY).toString();
|
|
970
|
+
if (currentText.length > 0) {
|
|
971
|
+
throw invalid('cloud snapshot attachment requires an empty local text file', 409);
|
|
796
972
|
}
|
|
797
|
-
|
|
973
|
+
const origin = { amalgmMutation: true, source: 'cloud-snapshot', patch: null };
|
|
974
|
+
Y.applyUpdate(entry.doc, new Uint8Array(bytes), origin);
|
|
975
|
+
persistDocState(Y, entry);
|
|
976
|
+
flushDocToDisk(Y, entry);
|
|
977
|
+
const registered = require('./replicas').registerReplica({
|
|
978
|
+
resourceId: input?.resourceId,
|
|
979
|
+
localResource: entry.resource,
|
|
980
|
+
localPath: entry.path,
|
|
981
|
+
contract: input?.contract,
|
|
982
|
+
schemaVersion: input?.schemaVersion,
|
|
983
|
+
authorityEpoch: input?.authorityEpoch,
|
|
984
|
+
appliedVersion: input?.snapshotVersion,
|
|
985
|
+
snapshotVersion: input?.snapshotVersion,
|
|
986
|
+
snapshotChecksum: checksum,
|
|
987
|
+
});
|
|
988
|
+
// File and SQLite binding are current before a mounted UI can observe the
|
|
989
|
+
// snapshot. Applying this full Yjs state is idempotent for an already-open
|
|
990
|
+
// recipient surface.
|
|
991
|
+
const event = appendStateEvent({
|
|
992
|
+
resource: entry.resource,
|
|
993
|
+
op: 'update',
|
|
994
|
+
id: entry.path,
|
|
995
|
+
patch: origin.patch || { yjs: snapshotBase64 },
|
|
996
|
+
sharedResourceId: input.resourceId,
|
|
997
|
+
source: 'doc:cloud-snapshot',
|
|
998
|
+
resourceVersion: Number(input.snapshotVersion),
|
|
999
|
+
});
|
|
1000
|
+
return { replica: registered.replica, event };
|
|
798
1001
|
}
|
|
799
1002
|
|
|
800
1003
|
/**
|
|
@@ -838,10 +1041,13 @@ function closeAllDocs() {
|
|
|
838
1041
|
|
|
839
1042
|
module.exports = {
|
|
840
1043
|
DOC_RESOURCE_PREFIX,
|
|
1044
|
+
applyCloudDocMutation,
|
|
1045
|
+
applyDocMutations,
|
|
841
1046
|
applyDocUpdates,
|
|
842
1047
|
closeAllDocs,
|
|
843
1048
|
docResourceName,
|
|
844
1049
|
getDocText,
|
|
1050
|
+
installCloudDocSnapshot,
|
|
845
1051
|
openDoc,
|
|
846
1052
|
pathFromDocResource,
|
|
847
1053
|
readDocResource,
|