mrmd-sync 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.js ADDED
@@ -0,0 +1,1127 @@
1
+ /**
2
+ * mrmd-sync - Production-ready sync server for mrmd
3
+ *
4
+ * Provides:
5
+ * - Yjs WebSocket sync for real-time collaboration
6
+ * - File system persistence with atomic writes
7
+ * - Yjs state persistence for crash recovery
8
+ * - Bidirectional sync: file changes ↔ Yjs updates
9
+ * - Structured logging, health checks, metrics
10
+ * - Graceful shutdown with pending write flush
11
+ */
12
+
13
+ import { WebSocketServer } from 'ws';
14
+ import http from 'http';
15
+ import * as Y from 'yjs';
16
+ import { diffChars } from 'diff';
17
+ import { watch } from 'chokidar';
18
+ import {
19
+ readFileSync,
20
+ writeFileSync,
21
+ renameSync,
22
+ unlinkSync,
23
+ existsSync,
24
+ mkdirSync,
25
+ statSync,
26
+ rmSync,
27
+ } from 'fs';
28
+ import { join, dirname, relative, resolve } from 'path';
29
+ import { createHash } from 'crypto';
30
+
31
+ // =============================================================================
32
+ // PID FILE - Prevents multiple instances on same directory
33
+ // =============================================================================
34
+
35
+ function acquirePidLock(lockDir, port, log) {
36
+ const pidFile = join(lockDir, 'server.pid');
37
+ const pid = process.pid;
38
+ const lockData = JSON.stringify({ pid, port, startedAt: new Date().toISOString() });
39
+
40
+ // Check if another instance is running
41
+ if (existsSync(pidFile)) {
42
+ try {
43
+ const existing = JSON.parse(readFileSync(pidFile, 'utf8'));
44
+
45
+ // Check if the process is still alive
46
+ try {
47
+ process.kill(existing.pid, 0); // Signal 0 = check if process exists
48
+ throw new Error(
49
+ `Another mrmd-sync instance is already running on this directory.\n\n` +
50
+ ` PID: ${existing.pid}\n` +
51
+ ` Port: ${existing.port}\n` +
52
+ ` Started: ${existing.startedAt}\n\n` +
53
+ `Stop the other instance first, or use a different directory.\n` +
54
+ `If the process is stale, delete: ${pidFile}`
55
+ );
56
+ } catch (err) {
57
+ if (err.code === 'ESRCH') {
58
+ // Process doesn't exist - stale PID file, safe to remove
59
+ log.info('Removing stale PID file', { stalePid: existing.pid });
60
+ unlinkSync(pidFile);
61
+ } else if (err.code !== 'EPERM') {
62
+ // EPERM means process exists but we can't signal it (still alive)
63
+ throw err;
64
+ } else {
65
+ // Process exists (EPERM) - another instance is running
66
+ throw new Error(
67
+ `Another mrmd-sync instance is already running on this directory.\n\n` +
68
+ ` PID: ${existing.pid}\n` +
69
+ ` Port: ${existing.port}\n` +
70
+ ` Started: ${existing.startedAt}\n\n` +
71
+ `Stop the other instance first, or use a different directory.`
72
+ );
73
+ }
74
+ }
75
+ } catch (err) {
76
+ if (err.message.includes('Another mrmd-sync')) {
77
+ throw err;
78
+ }
79
+ // Invalid JSON or other error - remove and recreate
80
+ log.warn('Invalid PID file, recreating', { error: err.message });
81
+ try { unlinkSync(pidFile); } catch { /* ignore */ }
82
+ }
83
+ }
84
+
85
+ // Write our PID
86
+ writeFileSync(pidFile, lockData);
87
+
88
+ return () => {
89
+ // Cleanup function - remove PID file on shutdown
90
+ try {
91
+ if (existsSync(pidFile)) {
92
+ const current = JSON.parse(readFileSync(pidFile, 'utf8'));
93
+ if (current.pid === pid) {
94
+ unlinkSync(pidFile);
95
+ }
96
+ }
97
+ } catch { /* ignore cleanup errors */ }
98
+ };
99
+ }
100
+
101
+ // y-websocket server utils
102
+ import * as syncProtocol from 'y-protocols/sync';
103
+ import * as awarenessProtocol from 'y-protocols/awareness';
104
+ import * as encoding from 'lib0/encoding';
105
+ import * as decoding from 'lib0/decoding';
106
+
107
+ const messageSync = 0;
108
+ const messageAwareness = 1;
109
+
110
+ // =============================================================================
111
+ // LOGGING - Structured JSON logging with levels
112
+ // =============================================================================
113
+
114
+ const LOG_LEVELS = { debug: 0, info: 1, warn: 2, error: 3 };
115
+
116
+ function createLogger(minLevel = 'info') {
117
+ const minLevelNum = LOG_LEVELS[minLevel] ?? 1;
118
+
119
+ const log = (level, message, data = {}) => {
120
+ if (LOG_LEVELS[level] < minLevelNum) return;
121
+
122
+ const entry = {
123
+ timestamp: new Date().toISOString(),
124
+ level,
125
+ message,
126
+ ...data,
127
+ };
128
+
129
+ const output = JSON.stringify(entry);
130
+ if (level === 'error') {
131
+ console.error(output);
132
+ } else {
133
+ console.log(output);
134
+ }
135
+ };
136
+
137
+ return {
138
+ debug: (msg, data) => log('debug', msg, data),
139
+ info: (msg, data) => log('info', msg, data),
140
+ warn: (msg, data) => log('warn', msg, data),
141
+ error: (msg, data) => log('error', msg, data),
142
+ };
143
+ }
144
+
145
+ // =============================================================================
146
+ // ASYNC MUTEX - Prevents race conditions in file operations
147
+ // =============================================================================
148
+
149
+ class AsyncMutex {
150
+ constructor() {
151
+ this._queue = [];
152
+ this._locked = false;
153
+ }
154
+
155
+ async acquire() {
156
+ return new Promise((resolve) => {
157
+ if (!this._locked) {
158
+ this._locked = true;
159
+ resolve();
160
+ } else {
161
+ this._queue.push(resolve);
162
+ }
163
+ });
164
+ }
165
+
166
+ release() {
167
+ if (this._queue.length > 0) {
168
+ const next = this._queue.shift();
169
+ next();
170
+ } else {
171
+ this._locked = false;
172
+ }
173
+ }
174
+
175
+ async withLock(fn) {
176
+ await this.acquire();
177
+ try {
178
+ return await fn();
179
+ } finally {
180
+ this.release();
181
+ }
182
+ }
183
+ }
184
+
185
+ // =============================================================================
186
+ // METRICS - Track server statistics
187
+ // =============================================================================
188
+
189
+ class Metrics {
190
+ constructor() {
191
+ this.startTime = Date.now();
192
+ this.totalConnections = 0;
193
+ this.activeConnections = 0;
194
+ this.totalMessages = 0;
195
+ this.totalBytesIn = 0;
196
+ this.totalBytesOut = 0;
197
+ this.totalFileSaves = 0;
198
+ this.totalFileLoads = 0;
199
+ this.totalErrors = 0;
200
+ this.lastActivity = Date.now();
201
+ }
202
+
203
+ connectionOpened() {
204
+ this.totalConnections++;
205
+ this.activeConnections++;
206
+ this.lastActivity = Date.now();
207
+ }
208
+
209
+ connectionClosed() {
210
+ this.activeConnections--;
211
+ }
212
+
213
+ messageReceived(bytes) {
214
+ this.totalMessages++;
215
+ this.totalBytesIn += bytes;
216
+ this.lastActivity = Date.now();
217
+ }
218
+
219
+ messageSent(bytes) {
220
+ this.totalBytesOut += bytes;
221
+ }
222
+
223
+ fileSaved() {
224
+ this.totalFileSaves++;
225
+ }
226
+
227
+ fileLoaded() {
228
+ this.totalFileLoads++;
229
+ }
230
+
231
+ errorOccurred() {
232
+ this.totalErrors++;
233
+ }
234
+
235
+ getStats() {
236
+ return {
237
+ uptime: Math.floor((Date.now() - this.startTime) / 1000),
238
+ connections: {
239
+ total: this.totalConnections,
240
+ active: this.activeConnections,
241
+ },
242
+ messages: {
243
+ total: this.totalMessages,
244
+ bytesIn: this.totalBytesIn,
245
+ bytesOut: this.totalBytesOut,
246
+ },
247
+ files: {
248
+ saves: this.totalFileSaves,
249
+ loads: this.totalFileLoads,
250
+ },
251
+ errors: this.totalErrors,
252
+ lastActivity: new Date(this.lastActivity).toISOString(),
253
+ };
254
+ }
255
+ }
256
+
257
+ // =============================================================================
258
+ // DEFAULT CONFIGURATION
259
+ // =============================================================================
260
+
261
+ const DEFAULT_CONFIG = {
262
+ dir: './docs',
263
+ port: 4444,
264
+ auth: null,
265
+ debounceMs: 1000,
266
+ maxConnections: 100,
267
+ maxConnectionsPerDoc: 50,
268
+ maxMessageSize: 1024 * 1024, // 1MB
269
+ pingIntervalMs: 30000,
270
+ docCleanupDelayMs: 60000,
271
+ maxFileSize: 10 * 1024 * 1024, // 10MB
272
+ dangerouslyAllowSystemPaths: false,
273
+ logLevel: 'info',
274
+ persistYjsState: true, // Save Yjs snapshots for crash recovery
275
+ snapshotIntervalMs: 30000, // Snapshot every 30s
276
+ };
277
+
278
+ // Paths that require dangerouslyAllowSystemPaths: true
279
+ const DANGEROUS_PATHS = ['/', '/etc', '/usr', '/var', '/bin', '/sbin', '/root', '/home'];
280
+
281
+ // =============================================================================
282
+ // UTILITY FUNCTIONS
283
+ // =============================================================================
284
+
285
+ /**
286
+ * Check if a directory path is considered dangerous
287
+ */
288
+ function isDangerousPath(dirPath) {
289
+ const resolved = dirPath.startsWith('/') ? dirPath : null;
290
+ if (!resolved) return false;
291
+
292
+ for (const dangerous of DANGEROUS_PATHS) {
293
+ if (resolved === dangerous || resolved.startsWith(dangerous + '/')) {
294
+ if (dangerous === '/home' && resolved.split('/').length > 3) {
295
+ return false;
296
+ }
297
+ if (dangerous !== '/home' && dangerous !== '/' && resolved !== dangerous) {
298
+ return true;
299
+ }
300
+ return true;
301
+ }
302
+ }
303
+ return false;
304
+ }
305
+
306
+ /**
307
+ * Safe file read with error handling
308
+ */
309
+ function safeReadFile(filePath, maxSize) {
310
+ try {
311
+ if (!existsSync(filePath)) {
312
+ return { content: null, error: null };
313
+ }
314
+ const stats = statSync(filePath);
315
+ if (stats.size > maxSize) {
316
+ return { content: null, error: `File too large: ${stats.size} bytes (max: ${maxSize})` };
317
+ }
318
+ const content = readFileSync(filePath, 'utf8');
319
+ return { content, error: null };
320
+ } catch (err) {
321
+ return { content: null, error: `Failed to read file: ${err.message}` };
322
+ }
323
+ }
324
+
325
+ /**
326
+ * Atomic file write - writes to temp file then renames
327
+ */
328
+ function atomicWriteFile(filePath, content) {
329
+ const tmpPath = `${filePath}.tmp.${process.pid}.${Date.now()}`;
330
+ try {
331
+ const fileDir = dirname(filePath);
332
+ mkdirSync(fileDir, { recursive: true });
333
+
334
+ // Write to temp file
335
+ writeFileSync(tmpPath, content, 'utf8');
336
+
337
+ // Atomic rename
338
+ renameSync(tmpPath, filePath);
339
+
340
+ return { success: true, error: null };
341
+ } catch (err) {
342
+ // Clean up temp file if it exists
343
+ try {
344
+ if (existsSync(tmpPath)) {
345
+ unlinkSync(tmpPath);
346
+ }
347
+ } catch {
348
+ // Ignore cleanup errors
349
+ }
350
+ return { success: false, error: `Failed to write file: ${err.message}` };
351
+ }
352
+ }
353
+
354
+ /**
355
+ * Save Yjs document state for crash recovery
356
+ */
357
+ function saveYjsSnapshot(snapshotPath, ydoc) {
358
+ try {
359
+ const state = Y.encodeStateAsUpdate(ydoc);
360
+ const buffer = Buffer.from(state);
361
+ atomicWriteFile(snapshotPath, buffer.toString('base64'));
362
+ return { success: true };
363
+ } catch (err) {
364
+ return { success: false, error: err.message };
365
+ }
366
+ }
367
+
368
+ /**
369
+ * Load Yjs document state from snapshot
370
+ */
371
+ function loadYjsSnapshot(snapshotPath, ydoc) {
372
+ try {
373
+ if (!existsSync(snapshotPath)) {
374
+ return { loaded: false };
375
+ }
376
+ const base64 = readFileSync(snapshotPath, 'utf8');
377
+ const state = Buffer.from(base64, 'base64');
378
+ Y.applyUpdate(ydoc, new Uint8Array(state));
379
+ return { loaded: true };
380
+ } catch (err) {
381
+ return { loaded: false, error: err.message };
382
+ }
383
+ }
384
+
385
+ /**
386
+ * Safe WebSocket send with error handling and metrics
387
+ */
388
+ function safeSend(ws, data, metrics) {
389
+ try {
390
+ if (ws.readyState === 1) {
391
+ ws.send(data);
392
+ if (metrics) {
393
+ metrics.messageSent(data.length);
394
+ }
395
+ return true;
396
+ }
397
+ } catch (err) {
398
+ // Silently fail - connection may be closing
399
+ }
400
+ return false;
401
+ }
402
+
403
+ /**
404
+ * Decode URL-encoded room name safely
405
+ */
406
+ function decodeRoomName(url) {
407
+ try {
408
+ const path = url?.slice(1).split('?')[0] || 'default';
409
+ return decodeURIComponent(path);
410
+ } catch {
411
+ return url?.slice(1).split('?')[0] || 'default';
412
+ }
413
+ }
414
+
415
+ /**
416
+ * Validate room/doc name
417
+ * Supports both relative names (within docs dir) and absolute paths
418
+ */
419
+ function isValidDocName(name) {
420
+ if (!name || typeof name !== 'string') return false;
421
+ if (name.length > 1024) return false; // Allow longer for absolute paths
422
+ if (name.includes('..')) return false; // No directory traversal
423
+
424
+ // Absolute paths are allowed (start with /)
425
+ if (name.startsWith('/')) {
426
+ // Must be a reasonable path - no control chars, null bytes, etc.
427
+ return /^\/[\w\-./]+$/.test(name);
428
+ }
429
+
430
+ // Relative names: alphanumeric, dashes, underscores, dots, slashes
431
+ if (name.startsWith('\\')) return false;
432
+ return /^[\w\-./]+$/.test(name);
433
+ }
434
+
435
+ /**
436
+ * Generate content hash for change detection
437
+ */
438
+ function contentHash(content) {
439
+ return createHash('md5').update(content).digest('hex');
440
+ }
441
+
442
+ // =============================================================================
443
+ // MAIN SERVER
444
+ // =============================================================================
445
+
446
+ /**
447
+ * Create a production-ready sync server
448
+ */
449
+ export function createServer(options = {}) {
450
+ const config = { ...DEFAULT_CONFIG, ...options };
451
+ const {
452
+ dir,
453
+ port,
454
+ auth,
455
+ debounceMs,
456
+ maxConnections,
457
+ maxConnectionsPerDoc,
458
+ maxMessageSize,
459
+ pingIntervalMs,
460
+ docCleanupDelayMs,
461
+ maxFileSize,
462
+ dangerouslyAllowSystemPaths,
463
+ logLevel,
464
+ persistYjsState,
465
+ snapshotIntervalMs,
466
+ } = config;
467
+
468
+ // Initialize logger and metrics
469
+ const log = createLogger(logLevel);
470
+ const metrics = new Metrics();
471
+
472
+ // Resolve directory path
473
+ const resolvedDir = resolve(dir);
474
+ const snapshotDir = join(resolvedDir, '.mrmd-sync');
475
+
476
+ // Security check
477
+ if (isDangerousPath(resolvedDir) && !dangerouslyAllowSystemPaths) {
478
+ throw new Error(
479
+ `Refusing to sync dangerous system path: "${resolvedDir}"\n\n` +
480
+ `Syncing system directories can expose sensitive files.\n\n` +
481
+ `To proceed, set: createServer({ dangerouslyAllowSystemPaths: true })\n` +
482
+ `Or use CLI: mrmd-sync --i-know-what-i-am-doing ${dir}`
483
+ );
484
+ }
485
+
486
+ // Ensure directories exist
487
+ mkdirSync(resolvedDir, { recursive: true });
488
+ mkdirSync(snapshotDir, { recursive: true }); // Always create for PID file
489
+
490
+ // Acquire PID lock to prevent multiple instances on same directory
491
+ const releasePidLock = acquirePidLock(snapshotDir, port, log);
492
+
493
+ // Document storage
494
+ const docs = new Map();
495
+
496
+ // Pending writes tracker for graceful shutdown
497
+ const pendingWrites = new Map();
498
+
499
+ // Shutdown state
500
+ let isShuttingDown = false;
501
+
502
+ // =============================================================================
503
+ // DOCUMENT MANAGEMENT
504
+ // =============================================================================
505
+
506
+ function getDoc(docName) {
507
+ if (docs.has(docName)) {
508
+ return docs.get(docName);
509
+ }
510
+
511
+ const ydoc = new Y.Doc();
512
+ ydoc.name = docName;
513
+
514
+ const awareness = new awarenessProtocol.Awareness(ydoc);
515
+ const conns = new Set();
516
+ const mutex = new AsyncMutex();
517
+
518
+ // Support absolute paths for files outside the docs directory
519
+ let filePath;
520
+ let isAbsolutePath = false;
521
+ if (docName.startsWith('/')) {
522
+ // Absolute path - use directly
523
+ isAbsolutePath = true;
524
+ filePath = docName.endsWith('.md') ? docName : `${docName}.md`;
525
+ } else {
526
+ // Relative path - join with docs directory
527
+ filePath = join(resolvedDir, docName.endsWith('.md') ? docName : `${docName}.md`);
528
+ }
529
+
530
+ // For snapshots, always use the snapshot dir with a safe name
531
+ const safeSnapshotName = docName.replace(/\//g, '__').replace(/^_+/, '');
532
+ const snapshotPath = persistYjsState
533
+ ? join(snapshotDir, `${safeSnapshotName}.yjs`)
534
+ : null;
535
+
536
+ const ytext = ydoc.getText('content');
537
+
538
+ // Track state
539
+ let lastFileHash = null;
540
+ let isWritingToFile = false;
541
+ let isWritingToYjs = false;
542
+ let writeTimeout = null;
543
+ let snapshotTimeout = null;
544
+ let cleanupTimeout = null;
545
+
546
+ // Log document opening
547
+ log.info('Opening document', {
548
+ doc: docName,
549
+ path: filePath,
550
+ absolute: isAbsolutePath,
551
+ });
552
+
553
+ // Try to load from Yjs snapshot first (for crash recovery)
554
+ let loadedFromSnapshot = false;
555
+ if (snapshotPath) {
556
+ const { loaded, error } = loadYjsSnapshot(snapshotPath, ydoc);
557
+ if (loaded) {
558
+ loadedFromSnapshot = true;
559
+ log.info('Loaded Yjs snapshot', { doc: docName });
560
+ } else if (error) {
561
+ log.warn('Failed to load Yjs snapshot', { doc: docName, error });
562
+ }
563
+ }
564
+
565
+ // Load from file if exists
566
+ const { content, error } = safeReadFile(filePath, maxFileSize);
567
+ if (error) {
568
+ log.error('Error loading file', { path: filePath, error });
569
+ metrics.errorOccurred();
570
+ } else if (content !== null) {
571
+ const currentContent = ytext.toString();
572
+ if (!loadedFromSnapshot || currentContent !== content) {
573
+ // File is source of truth if different from snapshot
574
+ if (currentContent !== content) {
575
+ ydoc.transact(() => {
576
+ if (ytext.length > 0) {
577
+ ytext.delete(0, ytext.length);
578
+ }
579
+ ytext.insert(0, content);
580
+ });
581
+ }
582
+ }
583
+ lastFileHash = contentHash(content);
584
+ log.info('Loaded file', { path: filePath, chars: content.length });
585
+ metrics.fileLoaded();
586
+ }
587
+
588
+ // Debounced write to file
589
+ const scheduleWrite = () => {
590
+ if (isWritingToYjs || isShuttingDown) return;
591
+
592
+ clearTimeout(writeTimeout);
593
+
594
+ const writeId = Date.now();
595
+ pendingWrites.set(docName, writeId);
596
+
597
+ writeTimeout = setTimeout(async () => {
598
+ await mutex.withLock(async () => {
599
+ if (isShuttingDown) return;
600
+
601
+ isWritingToFile = true;
602
+ const content = ytext.toString();
603
+ const hash = contentHash(content);
604
+
605
+ // Skip if content unchanged
606
+ if (hash === lastFileHash) {
607
+ isWritingToFile = false;
608
+ pendingWrites.delete(docName);
609
+ return;
610
+ }
611
+
612
+ const { success, error } = atomicWriteFile(filePath, content);
613
+ if (error) {
614
+ log.error('Error saving file', { path: filePath, error });
615
+ metrics.errorOccurred();
616
+ } else {
617
+ lastFileHash = hash;
618
+ log.info('Saved file', { path: filePath, chars: content.length });
619
+ metrics.fileSaved();
620
+ }
621
+
622
+ isWritingToFile = false;
623
+ pendingWrites.delete(docName);
624
+ });
625
+ }, debounceMs);
626
+ };
627
+
628
+ // Listen for Yjs updates
629
+ ydoc.on('update', scheduleWrite);
630
+
631
+ // Schedule Yjs snapshot saves
632
+ if (persistYjsState && snapshotPath) {
633
+ const scheduleSnapshot = () => {
634
+ clearTimeout(snapshotTimeout);
635
+ snapshotTimeout = setTimeout(() => {
636
+ const { success, error } = saveYjsSnapshot(snapshotPath, ydoc);
637
+ if (error) {
638
+ log.warn('Failed to save Yjs snapshot', { doc: docName, error });
639
+ }
640
+ scheduleSnapshot();
641
+ }, snapshotIntervalMs);
642
+ };
643
+ scheduleSnapshot();
644
+ }
645
+
646
+ // Apply external file changes to Yjs
647
+ const applyFileChange = async (newContent) => {
648
+ await mutex.withLock(() => {
649
+ if (isWritingToFile) return;
650
+
651
+ const newHash = contentHash(newContent);
652
+ if (newHash === lastFileHash) return;
653
+
654
+ const oldContent = ytext.toString();
655
+ if (oldContent === newContent) {
656
+ lastFileHash = newHash;
657
+ return;
658
+ }
659
+
660
+ isWritingToYjs = true;
661
+ const changes = diffChars(oldContent, newContent);
662
+
663
+ ydoc.transact(() => {
664
+ let pos = 0;
665
+ for (const change of changes) {
666
+ if (change.added) {
667
+ ytext.insert(pos, change.value);
668
+ pos += change.value.length;
669
+ } else if (change.removed) {
670
+ ytext.delete(pos, change.value.length);
671
+ } else {
672
+ pos += change.value.length;
673
+ }
674
+ }
675
+ });
676
+
677
+ lastFileHash = newHash;
678
+ log.info('Applied external file change', { path: filePath });
679
+ isWritingToYjs = false;
680
+ });
681
+ };
682
+
683
+ // Immediate flush for shutdown
684
+ const flushWrite = async () => {
685
+ clearTimeout(writeTimeout);
686
+ await mutex.withLock(async () => {
687
+ const content = ytext.toString();
688
+ const hash = contentHash(content);
689
+ if (hash !== lastFileHash) {
690
+ const { error } = atomicWriteFile(filePath, content);
691
+ if (error) {
692
+ log.error('Error flushing file', { path: filePath, error });
693
+ } else {
694
+ log.info('Flushed file on shutdown', { path: filePath });
695
+ }
696
+ }
697
+ // Save final snapshot
698
+ if (snapshotPath) {
699
+ saveYjsSnapshot(snapshotPath, ydoc);
700
+ }
701
+ });
702
+ pendingWrites.delete(docName);
703
+ };
704
+
705
+ // Cleanup
706
+ const scheduleCleanup = () => {
707
+ clearTimeout(cleanupTimeout);
708
+ cleanupTimeout = setTimeout(async () => {
709
+ if (conns.size === 0) {
710
+ await flushWrite();
711
+ clearTimeout(snapshotTimeout);
712
+ awareness.destroy();
713
+ ydoc.destroy();
714
+ docs.delete(docName);
715
+ log.info('Cleaned up document', { doc: docName });
716
+ }
717
+ }, docCleanupDelayMs);
718
+ };
719
+
720
+ const cancelCleanup = () => {
721
+ clearTimeout(cleanupTimeout);
722
+ };
723
+
724
+ const docData = {
725
+ ydoc,
726
+ ytext,
727
+ awareness,
728
+ conns,
729
+ mutex,
730
+ filePath,
731
+ snapshotPath,
732
+ applyFileChange,
733
+ flushWrite,
734
+ scheduleCleanup,
735
+ cancelCleanup,
736
+ };
737
+
738
+ docs.set(docName, docData);
739
+ return docData;
740
+ }
741
+
742
+ // =============================================================================
743
+ // FILE WATCHER
744
+ // =============================================================================
745
+
746
+ const watcher = watch(join(resolvedDir, '**/*.md'), {
747
+ ignoreInitial: true,
748
+ ignored: [join(snapshotDir, '**')], // Ignore snapshot directory
749
+ awaitWriteFinish: { stabilityThreshold: 300 },
750
+ });
751
+
752
+ watcher.on('change', async (filePath) => {
753
+ for (const [name, doc] of docs) {
754
+ if (doc.filePath === filePath) {
755
+ const { content, error } = safeReadFile(filePath, maxFileSize);
756
+ if (error) {
757
+ log.error('Error reading changed file', { path: filePath, error });
758
+ metrics.errorOccurred();
759
+ } else if (content !== null) {
760
+ await doc.applyFileChange(content);
761
+ }
762
+ break;
763
+ }
764
+ }
765
+ });
766
+
767
+ watcher.on('add', async (filePath) => {
768
+ const relativePath = relative(resolvedDir, filePath);
769
+ const docName = relativePath.replace(/\.md$/, '');
770
+
771
+ if (docs.has(docName)) {
772
+ const { content, error } = safeReadFile(filePath, maxFileSize);
773
+ if (error) {
774
+ log.error('Error reading new file', { path: filePath, error });
775
+ metrics.errorOccurred();
776
+ } else if (content !== null) {
777
+ await docs.get(docName).applyFileChange(content);
778
+ }
779
+ }
780
+ });
781
+
782
+ watcher.on('error', (err) => {
783
+ log.error('File watcher error', { error: err.message });
784
+ metrics.errorOccurred();
785
+ });
786
+
787
+ // =============================================================================
788
+ // HTTP SERVER WITH HEALTH ENDPOINT
789
+ // =============================================================================
790
+
791
+ const server = http.createServer((req, res) => {
792
+ const url = new URL(req.url, `http://${req.headers.host}`);
793
+
794
+ // CORS headers
795
+ res.setHeader('Access-Control-Allow-Origin', '*');
796
+ res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS');
797
+
798
+ if (req.method === 'OPTIONS') {
799
+ res.writeHead(204);
800
+ res.end();
801
+ return;
802
+ }
803
+
804
+ // Health check endpoint
805
+ if (url.pathname === '/health' || url.pathname === '/healthz') {
806
+ const healthy = !isShuttingDown && metrics.activeConnections >= 0;
807
+ res.writeHead(healthy ? 200 : 503, { 'Content-Type': 'application/json' });
808
+ res.end(JSON.stringify({
809
+ status: healthy ? 'healthy' : 'unhealthy',
810
+ shutting_down: isShuttingDown,
811
+ }));
812
+ return;
813
+ }
814
+
815
+ // Metrics endpoint
816
+ if (url.pathname === '/metrics') {
817
+ res.writeHead(200, { 'Content-Type': 'application/json' });
818
+ res.end(JSON.stringify(metrics.getStats(), null, 2));
819
+ return;
820
+ }
821
+
822
+ // Stats endpoint (detailed)
823
+ if (url.pathname === '/stats') {
824
+ const stats = {
825
+ ...metrics.getStats(),
826
+ documents: Array.from(docs.entries()).map(([name, doc]) => ({
827
+ name,
828
+ connections: doc.conns.size,
829
+ path: doc.filePath,
830
+ })),
831
+ config: {
832
+ port,
833
+ dir: resolvedDir,
834
+ debounceMs,
835
+ maxConnections,
836
+ maxConnectionsPerDoc,
837
+ },
838
+ };
839
+ res.writeHead(200, { 'Content-Type': 'application/json' });
840
+ res.end(JSON.stringify(stats, null, 2));
841
+ return;
842
+ }
843
+
844
+ // Default response
845
+ res.writeHead(200, { 'Content-Type': 'text/plain' });
846
+ res.end('mrmd-sync server\n\nEndpoints:\n /health - Health check\n /metrics - Server metrics\n /stats - Detailed statistics');
847
+ });
848
+
849
+ // =============================================================================
850
+ // WEBSOCKET SERVER
851
+ // =============================================================================
852
+
853
+ const wss = new WebSocketServer({
854
+ server,
855
+ maxPayload: maxMessageSize,
856
+ });
857
+
858
+ wss.on('connection', async (ws, req) => {
859
+ if (isShuttingDown) {
860
+ ws.close(1001, 'Server shutting down');
861
+ return;
862
+ }
863
+
864
+ if (metrics.activeConnections >= maxConnections) {
865
+ log.warn('Connection rejected: max connections reached');
866
+ ws.close(1013, 'Max connections reached');
867
+ return;
868
+ }
869
+
870
+ const docName = decodeRoomName(req.url);
871
+
872
+ if (!isValidDocName(docName)) {
873
+ log.warn('Connection rejected: invalid doc name', { docName });
874
+ ws.close(1008, 'Invalid document name');
875
+ return;
876
+ }
877
+
878
+ // Auth check
879
+ if (auth) {
880
+ try {
881
+ const authResult = await auth(req, docName);
882
+ if (!authResult) {
883
+ log.warn('Connection rejected: auth failed', { docName });
884
+ ws.close(1008, 'Unauthorized');
885
+ return;
886
+ }
887
+ } catch (err) {
888
+ log.error('Auth error', { error: err.message });
889
+ ws.close(1011, 'Auth error');
890
+ return;
891
+ }
892
+ }
893
+
894
+ const doc = getDoc(docName);
895
+ const { ydoc, awareness, conns, cancelCleanup, scheduleCleanup } = doc;
896
+
897
+ if (conns.size >= maxConnectionsPerDoc) {
898
+ log.warn('Connection rejected: max per-doc connections', { docName });
899
+ ws.close(1013, 'Max connections for document reached');
900
+ return;
901
+ }
902
+
903
+ metrics.connectionOpened();
904
+ conns.add(ws);
905
+ cancelCleanup();
906
+
907
+ log.info('Client connected', {
908
+ doc: docName,
909
+ clients: conns.size,
910
+ total: metrics.activeConnections,
911
+ });
912
+
913
+ // Heartbeat
914
+ let isAlive = true;
915
+ ws.on('pong', () => { isAlive = true; });
916
+
917
+ const pingInterval = setInterval(() => {
918
+ if (!isAlive) {
919
+ log.debug('Client timed out', { doc: docName });
920
+ ws.terminate();
921
+ return;
922
+ }
923
+ isAlive = false;
924
+ ws.ping();
925
+ }, pingIntervalMs);
926
+
927
+ // Send initial sync
928
+ const encoder = encoding.createEncoder();
929
+ encoding.writeVarUint(encoder, messageSync);
930
+ syncProtocol.writeSyncStep1(encoder, ydoc);
931
+ safeSend(ws, encoding.toUint8Array(encoder), metrics);
932
+
933
+ // Send awareness
934
+ const awarenessStates = awareness.getStates();
935
+ if (awarenessStates.size > 0) {
936
+ const awarenessEncoder = encoding.createEncoder();
937
+ encoding.writeVarUint(awarenessEncoder, messageAwareness);
938
+ encoding.writeVarUint8Array(
939
+ awarenessEncoder,
940
+ awarenessProtocol.encodeAwarenessUpdate(awareness, Array.from(awarenessStates.keys()))
941
+ );
942
+ safeSend(ws, encoding.toUint8Array(awarenessEncoder), metrics);
943
+ }
944
+
945
+ // Message handler
946
+ ws.on('message', (message) => {
947
+ try {
948
+ const data = new Uint8Array(message);
949
+ metrics.messageReceived(data.length);
950
+
951
+ const decoder = decoding.createDecoder(data);
952
+ const messageType = decoding.readVarUint(decoder);
953
+
954
+ switch (messageType) {
955
+ case messageSync: {
956
+ const responseEncoder = encoding.createEncoder();
957
+ encoding.writeVarUint(responseEncoder, messageSync);
958
+ syncProtocol.readSyncMessage(decoder, responseEncoder, ydoc, ws);
959
+
960
+ if (encoding.length(responseEncoder) > 1) {
961
+ safeSend(ws, encoding.toUint8Array(responseEncoder), metrics);
962
+ }
963
+ break;
964
+ }
965
+ case messageAwareness: {
966
+ awarenessProtocol.applyAwarenessUpdate(
967
+ awareness,
968
+ decoding.readVarUint8Array(decoder),
969
+ ws
970
+ );
971
+ break;
972
+ }
973
+ }
974
+ } catch (err) {
975
+ log.error('Error processing message', { error: err.message, doc: docName });
976
+ metrics.errorOccurred();
977
+ }
978
+ });
979
+
980
+ // Broadcast updates
981
+ const updateHandler = (update, origin) => {
982
+ const broadcastEncoder = encoding.createEncoder();
983
+ encoding.writeVarUint(broadcastEncoder, messageSync);
984
+ syncProtocol.writeUpdate(broadcastEncoder, update);
985
+ const msg = encoding.toUint8Array(broadcastEncoder);
986
+
987
+ conns.forEach((conn) => {
988
+ if (conn !== origin) {
989
+ safeSend(conn, msg, metrics);
990
+ }
991
+ });
992
+ };
993
+ ydoc.on('update', updateHandler);
994
+
995
+ // Broadcast awareness
996
+ const awarenessHandler = ({ added, updated, removed }) => {
997
+ const changedClients = added.concat(updated).concat(removed);
998
+ const awarenessEncoder = encoding.createEncoder();
999
+ encoding.writeVarUint(awarenessEncoder, messageAwareness);
1000
+ encoding.writeVarUint8Array(
1001
+ awarenessEncoder,
1002
+ awarenessProtocol.encodeAwarenessUpdate(awareness, changedClients)
1003
+ );
1004
+ const msg = encoding.toUint8Array(awarenessEncoder);
1005
+ conns.forEach((conn) => safeSend(conn, msg, metrics));
1006
+ };
1007
+ awareness.on('update', awarenessHandler);
1008
+
1009
+ // Cleanup on close
1010
+ ws.on('close', () => {
1011
+ clearInterval(pingInterval);
1012
+ metrics.connectionClosed();
1013
+ conns.delete(ws);
1014
+ ydoc.off('update', updateHandler);
1015
+ awareness.off('update', awarenessHandler);
1016
+ awarenessProtocol.removeAwarenessStates(awareness, [ydoc.clientID], null);
1017
+
1018
+ log.info('Client disconnected', {
1019
+ doc: docName,
1020
+ clients: conns.size,
1021
+ total: metrics.activeConnections,
1022
+ });
1023
+
1024
+ if (conns.size === 0) {
1025
+ scheduleCleanup();
1026
+ }
1027
+ });
1028
+
1029
+ ws.on('error', (err) => {
1030
+ log.error('WebSocket error', { error: err.message, doc: docName });
1031
+ metrics.errorOccurred();
1032
+ });
1033
+ });
1034
+
1035
+ wss.on('error', (err) => {
1036
+ log.error('WebSocket server error', { error: err.message });
1037
+ metrics.errorOccurred();
1038
+ });
1039
+
1040
+ // =============================================================================
1041
+ // GRACEFUL SHUTDOWN
1042
+ // =============================================================================
1043
+
1044
+ const gracefulShutdown = async (signal) => {
1045
+ if (isShuttingDown) return;
1046
+ isShuttingDown = true;
1047
+
1048
+ log.info('Graceful shutdown initiated', { signal });
1049
+
1050
+ // Stop accepting new connections
1051
+ wss.clients.forEach((client) => {
1052
+ client.close(1001, 'Server shutting down');
1053
+ });
1054
+
1055
+ // Flush all pending writes
1056
+ log.info('Flushing pending writes', { count: docs.size });
1057
+ const flushPromises = Array.from(docs.values()).map((doc) => doc.flushWrite());
1058
+ await Promise.all(flushPromises);
1059
+
1060
+ // Close watcher
1061
+ await watcher.close();
1062
+
1063
+ // Close servers
1064
+ wss.close();
1065
+ server.close();
1066
+
1067
+ // Cleanup docs
1068
+ docs.forEach((doc) => {
1069
+ doc.awareness.destroy();
1070
+ doc.ydoc.destroy();
1071
+ });
1072
+ docs.clear();
1073
+
1074
+ // Release PID lock
1075
+ releasePidLock();
1076
+
1077
+ log.info('Shutdown complete');
1078
+ };
1079
+
1080
+ // Register shutdown handlers
1081
+ process.on('SIGTERM', () => gracefulShutdown('SIGTERM'));
1082
+ process.on('SIGINT', () => gracefulShutdown('SIGINT'));
1083
+
1084
+ // =============================================================================
1085
+ // START SERVER
1086
+ // =============================================================================
1087
+
1088
+ server.listen(port, '0.0.0.0', () => {
1089
+ log.info('Server started', {
1090
+ port,
1091
+ dir: resolvedDir,
1092
+ debounceMs,
1093
+ maxConnections,
1094
+ persistYjsState,
1095
+ });
1096
+ });
1097
+
1098
+ // =============================================================================
1099
+ // PUBLIC API
1100
+ // =============================================================================
1101
+
1102
+ return {
1103
+ server,
1104
+ wss,
1105
+ docs,
1106
+ getDoc,
1107
+ config,
1108
+ metrics,
1109
+ log,
1110
+
1111
+ getStats() {
1112
+ return {
1113
+ ...metrics.getStats(),
1114
+ docs: Array.from(docs.entries()).map(([name, doc]) => ({
1115
+ name,
1116
+ connections: doc.conns.size,
1117
+ })),
1118
+ };
1119
+ },
1120
+
1121
+ async close() {
1122
+ await gracefulShutdown('API');
1123
+ },
1124
+ };
1125
+ }
1126
+
1127
+ export default { createServer };