@lark-apaas/fullstack-cli 1.1.59-alpha.20260720184331 → 1.1.59-alpha.20260721044121
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
const fs = require('fs');
|
|
4
4
|
const path = require('path');
|
|
5
|
+
const { spawnSync: nodeSpawnSync } = require('child_process');
|
|
5
6
|
|
|
6
7
|
const DEFAULT_OUTBOX_ROOT = '/tmp/event/miaoda-preview-phase/v1';
|
|
7
8
|
const MAX_EVENT_BYTES = 16 * 1024;
|
|
@@ -10,10 +11,186 @@ const DEFAULT_MAX_ROOT_BYTES = 64 * 1024 * 1024;
|
|
|
10
11
|
const DEFAULT_MAX_ROOT_DIRECTORIES = 128;
|
|
11
12
|
const DEFAULT_MAX_ROOT_SCAN_DIRECTORIES = 256;
|
|
12
13
|
const DEFAULT_MAX_RUN_SCAN_ENTRIES = 64;
|
|
14
|
+
const DEFAULT_MAX_FALLBACK_FILES = 32;
|
|
15
|
+
const DEFAULT_MAX_FALLBACK_BYTES = 2 * 1024 * 1024;
|
|
16
|
+
const DEFAULT_MAX_FALLBACK_EVENTS = 256;
|
|
17
|
+
const MAX_FALLBACK_RECORD_BYTES = MAX_EVENT_BYTES + 4 * 1024;
|
|
18
|
+
const FALLBACK_TERMINAL_RESERVE_BYTES = 1024;
|
|
13
19
|
const RUN_ID_PATTERN = /^prv_[A-Za-z0-9_-]{1,60}$/;
|
|
14
20
|
const STATE_ID_PATTERN = /^[A-Za-z0-9_-]{1,128}$/;
|
|
21
|
+
const PRODUCER_INSTANCE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
|
22
|
+
const FALLBACK_SESSION_ID_PATTERN = /^pfs_[a-f0-9]{32}$/;
|
|
15
23
|
const PROCESS_START_TICKS_PATTERN = /^\d+$/;
|
|
16
24
|
const PRODUCER_EPOCH_PATTERN = /^ppe_[a-f0-9]{32}$/;
|
|
25
|
+
const INSTANCE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/;
|
|
26
|
+
const PRODUCER_REGISTER_URL = 'http://127.0.0.1:8091/shell/exec';
|
|
27
|
+
const PRODUCER_REGISTER_ATTEMPTS = 2;
|
|
28
|
+
const PRODUCER_REGISTER_CURL_TIMEOUT_SECONDS = '0.35';
|
|
29
|
+
const PRODUCER_REGISTER_PROCESS_TIMEOUT_MS = 400;
|
|
30
|
+
const PRODUCER_REGISTER_MAX_RESPONSE_BYTES = 64 * 1024;
|
|
31
|
+
const TRANSIENT_ERRNO_CODES = new Set([
|
|
32
|
+
'EAGAIN',
|
|
33
|
+
'EINTR',
|
|
34
|
+
'EBUSY',
|
|
35
|
+
'EMFILE',
|
|
36
|
+
'ENFILE',
|
|
37
|
+
'ENOMEM',
|
|
38
|
+
]);
|
|
39
|
+
const MAX_TRANSIENT_ATTEMPTS = 3;
|
|
40
|
+
|
|
41
|
+
function producerRegistrationError(code, message, cause) {
|
|
42
|
+
const error = new Error(message, cause == null ? undefined : { cause });
|
|
43
|
+
error.code = code;
|
|
44
|
+
return error;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function validateProducerRegistrationResponse(response, request) {
|
|
48
|
+
const commonInvalid =
|
|
49
|
+
response?.protocol !== 'preview-phase-outbox-v1' ||
|
|
50
|
+
response.capability_status !== 'supported' ||
|
|
51
|
+
response.generation !== request.generation ||
|
|
52
|
+
response.producer_epoch !== request.producer_epoch ||
|
|
53
|
+
response.producer_generation !== request.generation;
|
|
54
|
+
const rejected = response?.registration_status === 'rejected';
|
|
55
|
+
const rejectedInvalid =
|
|
56
|
+
rejected &&
|
|
57
|
+
(response.registration_reason !== 'stale_epoch' ||
|
|
58
|
+
response.writer_active === true ||
|
|
59
|
+
response.sealed !== true ||
|
|
60
|
+
response.fallback_required !== true ||
|
|
61
|
+
response.fallback_session_id !== request.fallback_session_id ||
|
|
62
|
+
response.fallback_final_seq !== 1 ||
|
|
63
|
+
response.producer_state !== 'rejected' ||
|
|
64
|
+
request.rejected_event == null);
|
|
65
|
+
const acceptedInvalid =
|
|
66
|
+
!rejected &&
|
|
67
|
+
((response?.registration_status != null &&
|
|
68
|
+
response.registration_status !== 'accepted') ||
|
|
69
|
+
response.writer_active !== true ||
|
|
70
|
+
response.producer_state !== 'active');
|
|
71
|
+
if (commonInvalid || rejectedInvalid || acceptedInvalid) {
|
|
72
|
+
throw producerRegistrationError(
|
|
73
|
+
'PREVIEW_PHASE_PRODUCER_REGISTRATION_ACK_UNKNOWN',
|
|
74
|
+
'invalid preview phase producer registration response'
|
|
75
|
+
);
|
|
76
|
+
}
|
|
77
|
+
return response;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function registerPreviewPhaseProducerViaShellServer(
|
|
81
|
+
request,
|
|
82
|
+
spawnSync = nodeSpawnSync
|
|
83
|
+
) {
|
|
84
|
+
const wireRequest = {
|
|
85
|
+
phase_outbox: {
|
|
86
|
+
protocol: 'preview-phase-outbox-v1',
|
|
87
|
+
operation: 'register',
|
|
88
|
+
run_id: request.run_id,
|
|
89
|
+
generation: request.generation,
|
|
90
|
+
expected_instance_id: request.expected_instance_id,
|
|
91
|
+
producer_epoch: request.producer_epoch,
|
|
92
|
+
writer_instance_id: request.writer_instance_id,
|
|
93
|
+
pid: request.pid,
|
|
94
|
+
process_start_ticks: request.process_start_ticks,
|
|
95
|
+
...(request.rejected_event == null
|
|
96
|
+
? {}
|
|
97
|
+
: {
|
|
98
|
+
producer_instance_id: request.producer_instance_id,
|
|
99
|
+
fallback_session_id: request.fallback_session_id,
|
|
100
|
+
rejected_event: request.rejected_event,
|
|
101
|
+
}),
|
|
102
|
+
},
|
|
103
|
+
};
|
|
104
|
+
const result = spawnSync(
|
|
105
|
+
'curl',
|
|
106
|
+
[
|
|
107
|
+
'--silent',
|
|
108
|
+
'--show-error',
|
|
109
|
+
'--connect-timeout',
|
|
110
|
+
PRODUCER_REGISTER_CURL_TIMEOUT_SECONDS,
|
|
111
|
+
'--max-time',
|
|
112
|
+
PRODUCER_REGISTER_CURL_TIMEOUT_SECONDS,
|
|
113
|
+
'--request',
|
|
114
|
+
'POST',
|
|
115
|
+
'--header',
|
|
116
|
+
'Content-Type: application/json',
|
|
117
|
+
'--data-binary',
|
|
118
|
+
'@-',
|
|
119
|
+
PRODUCER_REGISTER_URL,
|
|
120
|
+
],
|
|
121
|
+
{
|
|
122
|
+
input: Buffer.from(JSON.stringify(wireRequest), 'utf8'),
|
|
123
|
+
encoding: 'utf8',
|
|
124
|
+
timeout: PRODUCER_REGISTER_PROCESS_TIMEOUT_MS,
|
|
125
|
+
maxBuffer: PRODUCER_REGISTER_MAX_RESPONSE_BYTES,
|
|
126
|
+
windowsHide: true,
|
|
127
|
+
}
|
|
128
|
+
);
|
|
129
|
+
if (result?.error) {
|
|
130
|
+
const code =
|
|
131
|
+
result.error.code === 'ETIMEDOUT'
|
|
132
|
+
? 'PREVIEW_PHASE_PRODUCER_REGISTRATION_ACK_UNKNOWN'
|
|
133
|
+
: 'PREVIEW_PHASE_PRODUCER_REGISTRATION_UNAVAILABLE';
|
|
134
|
+
throw producerRegistrationError(
|
|
135
|
+
code,
|
|
136
|
+
`preview phase producer registration transport failed: ${result.error.message}`,
|
|
137
|
+
result.error
|
|
138
|
+
);
|
|
139
|
+
}
|
|
140
|
+
if (result?.signal || result?.status === 28) {
|
|
141
|
+
throw producerRegistrationError(
|
|
142
|
+
'PREVIEW_PHASE_PRODUCER_REGISTRATION_ACK_UNKNOWN',
|
|
143
|
+
'preview phase producer registration acknowledgement timed out'
|
|
144
|
+
);
|
|
145
|
+
}
|
|
146
|
+
if (result?.status !== 0) {
|
|
147
|
+
throw producerRegistrationError(
|
|
148
|
+
'PREVIEW_PHASE_PRODUCER_REGISTRATION_UNAVAILABLE',
|
|
149
|
+
`preview phase producer registration transport exited ${String(result?.status)}`
|
|
150
|
+
);
|
|
151
|
+
}
|
|
152
|
+
let envelope;
|
|
153
|
+
try {
|
|
154
|
+
envelope = JSON.parse(result.stdout || '');
|
|
155
|
+
} catch (error) {
|
|
156
|
+
throw producerRegistrationError(
|
|
157
|
+
'PREVIEW_PHASE_PRODUCER_REGISTRATION_ACK_UNKNOWN',
|
|
158
|
+
'preview phase producer registration returned invalid JSON',
|
|
159
|
+
error
|
|
160
|
+
);
|
|
161
|
+
}
|
|
162
|
+
if (envelope?.success !== true || envelope?.data?.success !== true) {
|
|
163
|
+
const detail =
|
|
164
|
+
envelope?.data?.error || envelope?.error || envelope?.message || 'rejected';
|
|
165
|
+
const detailText = String(detail);
|
|
166
|
+
let code = 'PREVIEW_PHASE_PRODUCER_REGISTRATION_REJECTED';
|
|
167
|
+
if (detailText.includes('registration acknowledgement unknown')) {
|
|
168
|
+
code = 'PREVIEW_PHASE_PRODUCER_REGISTRATION_ACK_UNKNOWN';
|
|
169
|
+
} else if (detailText.includes('producer epoch already belongs')) {
|
|
170
|
+
code = 'PREVIEW_PHASE_PRODUCER_EPOCH_CONFLICT';
|
|
171
|
+
} else if (
|
|
172
|
+
detailText.includes('publish preview phase canonical registration') ||
|
|
173
|
+
detailText.includes('conflicts with canonical run')
|
|
174
|
+
) {
|
|
175
|
+
code = 'PREVIEW_PHASE_RUN_CONFLICT';
|
|
176
|
+
}
|
|
177
|
+
throw producerRegistrationError(
|
|
178
|
+
code,
|
|
179
|
+
`preview phase producer registration rejected: ${detailText}`
|
|
180
|
+
);
|
|
181
|
+
}
|
|
182
|
+
let response;
|
|
183
|
+
try {
|
|
184
|
+
response = JSON.parse(envelope.data.output || '');
|
|
185
|
+
} catch (error) {
|
|
186
|
+
throw producerRegistrationError(
|
|
187
|
+
'PREVIEW_PHASE_PRODUCER_REGISTRATION_ACK_UNKNOWN',
|
|
188
|
+
'preview phase producer registration acknowledgement was invalid',
|
|
189
|
+
error
|
|
190
|
+
);
|
|
191
|
+
}
|
|
192
|
+
return validateProducerRegistrationResponse(response, request);
|
|
193
|
+
}
|
|
17
194
|
|
|
18
195
|
function readLinuxProcessStartTicks(fileSystem, pid) {
|
|
19
196
|
try {
|
|
@@ -47,13 +224,46 @@ function assertSafeDirectory(fileSystem, directoryPath) {
|
|
|
47
224
|
fileSystem.chmodSync(directoryPath, 0o700);
|
|
48
225
|
}
|
|
49
226
|
|
|
227
|
+
// Canonical generations are published only by the staging-directory rename.
|
|
228
|
+
// Every later lookup must be existing-only: recreating a path that the
|
|
229
|
+
// controller just quarantined would hide the durable generation and could
|
|
230
|
+
// manufacture a second, unrelated canonical lifecycle.
|
|
231
|
+
function assertExistingSafeDirectory(fileSystem, directoryPath) {
|
|
232
|
+
const stat = fileSystem.lstatSync(directoryPath);
|
|
233
|
+
if (!stat.isDirectory() || stat.isSymbolicLink()) {
|
|
234
|
+
throw new Error(`unsafe preview phase outbox path: ${directoryPath}`);
|
|
235
|
+
}
|
|
236
|
+
fileSystem.chmodSync(directoryPath, 0o700);
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
function createFallbackSessionID(randomUUID = require('crypto').randomUUID) {
|
|
240
|
+
const sessionID = `pfs_${String(randomUUID())
|
|
241
|
+
.replaceAll('-', '')
|
|
242
|
+
.toLowerCase()}`;
|
|
243
|
+
if (!FALLBACK_SESSION_ID_PATTERN.test(sessionID)) {
|
|
244
|
+
throw new Error('invalid preview phase fallback session UUID');
|
|
245
|
+
}
|
|
246
|
+
return sessionID;
|
|
247
|
+
}
|
|
248
|
+
|
|
50
249
|
function createPreviewPhaseOutbox(options) {
|
|
51
250
|
const fileSystem = options.fs || fs;
|
|
52
251
|
const root = options.root || DEFAULT_OUTBOX_ROOT;
|
|
53
252
|
const randomUUID = options.random_uuid || require('crypto').randomUUID;
|
|
54
253
|
const now = options.now || Date.now;
|
|
55
254
|
const writerInstanceID = options.writer_instance_id || randomUUID();
|
|
255
|
+
const producerInstanceID = options.producer_instance_id || writerInstanceID;
|
|
256
|
+
const fallbackSessionID =
|
|
257
|
+
options.fallback_session_id || createFallbackSessionID();
|
|
56
258
|
const producerEpoch = options.producer_epoch;
|
|
259
|
+
const expectedInstanceID = options.expected_instance_id;
|
|
260
|
+
const registerProducer =
|
|
261
|
+
options.register_producer ||
|
|
262
|
+
(request =>
|
|
263
|
+
registerPreviewPhaseProducerViaShellServer(
|
|
264
|
+
request,
|
|
265
|
+
options.spawn_sync || nodeSpawnSync
|
|
266
|
+
));
|
|
57
267
|
const getProcessStartTicks =
|
|
58
268
|
options.get_process_start_ticks ||
|
|
59
269
|
(pid => readLinuxProcessStartTicks(fileSystem, pid));
|
|
@@ -70,15 +280,36 @@ function createPreviewPhaseOutbox(options) {
|
|
|
70
280
|
options.max_root_scan_directories ?? DEFAULT_MAX_ROOT_SCAN_DIRECTORIES;
|
|
71
281
|
const maxRunScanEntries =
|
|
72
282
|
options.max_run_scan_entries ?? DEFAULT_MAX_RUN_SCAN_ENTRIES;
|
|
283
|
+
const maxFallbackFiles =
|
|
284
|
+
options.max_fallback_files ??
|
|
285
|
+
Math.min(DEFAULT_MAX_FALLBACK_FILES, maxRunScanEntries);
|
|
286
|
+
const maxFallbackBytes =
|
|
287
|
+
options.max_fallback_bytes ?? DEFAULT_MAX_FALLBACK_BYTES;
|
|
288
|
+
const maxFallbackEvents =
|
|
289
|
+
options.max_fallback_events ?? DEFAULT_MAX_FALLBACK_EVENTS;
|
|
290
|
+
const maxFallbackRecordBytes =
|
|
291
|
+
options.max_fallback_record_bytes ?? MAX_FALLBACK_RECORD_BYTES;
|
|
73
292
|
if (!RUN_ID_PATTERN.test(options.run_id || '')) {
|
|
74
293
|
throw new Error('invalid preview run_id for phase outbox');
|
|
75
294
|
}
|
|
76
295
|
if (!PRODUCER_EPOCH_PATTERN.test(producerEpoch || '')) {
|
|
77
296
|
throw new Error('invalid preview phase producer_epoch');
|
|
78
297
|
}
|
|
298
|
+
if (!INSTANCE_ID_PATTERN.test(expectedInstanceID || '')) {
|
|
299
|
+
throw new Error('invalid preview phase expected_instance_id');
|
|
300
|
+
}
|
|
301
|
+
if (typeof registerProducer !== 'function') {
|
|
302
|
+
throw new Error('invalid preview phase producer registrar');
|
|
303
|
+
}
|
|
79
304
|
if (!STATE_ID_PATTERN.test(writerInstanceID)) {
|
|
80
305
|
throw new Error('invalid preview phase writer_instance_id');
|
|
81
306
|
}
|
|
307
|
+
if (!STATE_ID_PATTERN.test(producerInstanceID)) {
|
|
308
|
+
throw new Error('invalid preview phase producer_instance_id');
|
|
309
|
+
}
|
|
310
|
+
if (!FALLBACK_SESSION_ID_PATTERN.test(fallbackSessionID)) {
|
|
311
|
+
throw new Error('invalid preview phase fallback_session_id');
|
|
312
|
+
}
|
|
82
313
|
if (!PROCESS_START_TICKS_PATTERN.test(currentProcessStartTicks || '')) {
|
|
83
314
|
throw new Error('invalid preview phase process_start_ticks');
|
|
84
315
|
}
|
|
@@ -100,13 +331,41 @@ function createPreviewPhaseOutbox(options) {
|
|
|
100
331
|
if (!Number.isSafeInteger(maxRunScanEntries) || maxRunScanEntries <= 0) {
|
|
101
332
|
throw new Error('invalid preview phase max_run_scan_entries');
|
|
102
333
|
}
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
334
|
+
if (
|
|
335
|
+
!Number.isSafeInteger(maxFallbackFiles) ||
|
|
336
|
+
maxFallbackFiles <= 0 ||
|
|
337
|
+
maxFallbackFiles > maxRunScanEntries
|
|
338
|
+
) {
|
|
339
|
+
throw new Error('invalid preview phase max_fallback_files');
|
|
340
|
+
}
|
|
341
|
+
if (!Number.isSafeInteger(maxFallbackBytes) || maxFallbackBytes <= 0) {
|
|
342
|
+
throw new Error('invalid preview phase max_fallback_bytes');
|
|
343
|
+
}
|
|
344
|
+
if (!Number.isSafeInteger(maxFallbackEvents) || maxFallbackEvents <= 0) {
|
|
345
|
+
throw new Error('invalid preview phase max_fallback_events');
|
|
346
|
+
}
|
|
347
|
+
if (
|
|
348
|
+
!Number.isSafeInteger(maxFallbackRecordBytes) ||
|
|
349
|
+
maxFallbackRecordBytes <= 0
|
|
350
|
+
) {
|
|
351
|
+
throw new Error('invalid preview phase max_fallback_record_bytes');
|
|
352
|
+
}
|
|
353
|
+
const canonicalRunDir = path.join(root, options.run_id);
|
|
354
|
+
const stagingName = `.staging-${process.pid}-${writerInstanceID}`;
|
|
355
|
+
const stagingRunDir = path.join(root, stagingName);
|
|
356
|
+
let runDir = canonicalRunDir;
|
|
357
|
+
let eventsPath = path.join(runDir, 'events.jsonl');
|
|
358
|
+
let metaPath = path.join(runDir, 'meta.json');
|
|
359
|
+
let writerLockPath = path.join(runDir, 'writer.lock');
|
|
360
|
+
let sealPath = path.join(runDir, 'sealed.json');
|
|
361
|
+
let degradedPath = path.join(runDir, 'degraded.json');
|
|
362
|
+
let fallbackIntentPath = path.join(runDir, 'fallback.intent.json');
|
|
363
|
+
let quiesceRequestPath = path.join(runDir, 'quiesce.request.json');
|
|
364
|
+
let unboundFallbackPath = path.join(runDir, 'producer.unbound.json');
|
|
365
|
+
const fallbackSpoolPath = path.join(
|
|
366
|
+
canonicalRunDir,
|
|
367
|
+
`fallback-${fallbackSessionID}.jsonl`
|
|
368
|
+
);
|
|
110
369
|
const producerEpochMarkerPath = path.join(
|
|
111
370
|
root,
|
|
112
371
|
`producer-epoch-${producerEpoch}.json`
|
|
@@ -114,6 +373,7 @@ function createPreviewPhaseOutbox(options) {
|
|
|
114
373
|
let recovered = false;
|
|
115
374
|
let metadata = null;
|
|
116
375
|
let writerLockOwned = false;
|
|
376
|
+
let writerLockDirectory = null;
|
|
117
377
|
let eventsFD = null;
|
|
118
378
|
let committedBytes = null;
|
|
119
379
|
let degraded = false;
|
|
@@ -123,8 +383,49 @@ function createPreviewPhaseOutbox(options) {
|
|
|
123
383
|
let closeResult = false;
|
|
124
384
|
let producerMarkerActive = false;
|
|
125
385
|
let producerMarkerTerminal = false;
|
|
386
|
+
let producerRegistrationDurable = false;
|
|
387
|
+
let producerRegistrationAttemptCount = 0;
|
|
388
|
+
let producerRegistrationLastError = null;
|
|
126
389
|
let producerStartedAtMs = null;
|
|
127
390
|
let producerTerminalAtMs = null;
|
|
391
|
+
let canonicalPublicationDurable = false;
|
|
392
|
+
let everOwnedCanonicalWriter = false;
|
|
393
|
+
let fallbackFD = null;
|
|
394
|
+
let fallbackBytes = 0;
|
|
395
|
+
let fallbackSeq = 0;
|
|
396
|
+
let fallbackStarted = false;
|
|
397
|
+
let fallbackSealed = false;
|
|
398
|
+
let fallbackFailed = false;
|
|
399
|
+
let fallbackCreatedAt = null;
|
|
400
|
+
function selectRunDirectory(directoryPath) {
|
|
401
|
+
if (runDir !== directoryPath) {
|
|
402
|
+
if (eventsFD != null) {
|
|
403
|
+
throw new Error(
|
|
404
|
+
'cannot switch preview phase directory with an open events fd'
|
|
405
|
+
);
|
|
406
|
+
}
|
|
407
|
+
metadata = null;
|
|
408
|
+
recovered = false;
|
|
409
|
+
committedBytes = null;
|
|
410
|
+
}
|
|
411
|
+
runDir = directoryPath;
|
|
412
|
+
eventsPath = path.join(runDir, 'events.jsonl');
|
|
413
|
+
metaPath = path.join(runDir, 'meta.json');
|
|
414
|
+
writerLockPath = path.join(runDir, 'writer.lock');
|
|
415
|
+
sealPath = path.join(runDir, 'sealed.json');
|
|
416
|
+
degradedPath = path.join(runDir, 'degraded.json');
|
|
417
|
+
fallbackIntentPath = path.join(runDir, 'fallback.intent.json');
|
|
418
|
+
quiesceRequestPath = path.join(runDir, 'quiesce.request.json');
|
|
419
|
+
unboundFallbackPath = path.join(runDir, 'producer.unbound.json');
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
function ownsSelectedWriterLock() {
|
|
423
|
+
return writerLockOwned && writerLockDirectory === runDir;
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
function isTransientFilesystemError(error) {
|
|
427
|
+
return TRANSIENT_ERRNO_CODES.has(error?.code);
|
|
428
|
+
}
|
|
128
429
|
|
|
129
430
|
function rootQuotaError(message) {
|
|
130
431
|
const error = new Error(message);
|
|
@@ -245,12 +546,12 @@ function createPreviewPhaseOutbox(options) {
|
|
|
245
546
|
return value;
|
|
246
547
|
}
|
|
247
548
|
|
|
248
|
-
function
|
|
549
|
+
function syncDirectory(directoryPath) {
|
|
249
550
|
const flags =
|
|
250
551
|
fs.constants.O_RDONLY |
|
|
251
552
|
(fs.constants.O_DIRECTORY || 0) |
|
|
252
553
|
(fs.constants.O_NOFOLLOW || 0);
|
|
253
|
-
const fd = fileSystem.openSync(
|
|
554
|
+
const fd = fileSystem.openSync(directoryPath, flags);
|
|
254
555
|
try {
|
|
255
556
|
fileSystem.fsyncSync(fd);
|
|
256
557
|
} finally {
|
|
@@ -258,6 +559,10 @@ function createPreviewPhaseOutbox(options) {
|
|
|
258
559
|
}
|
|
259
560
|
}
|
|
260
561
|
|
|
562
|
+
function syncRunDirectory() {
|
|
563
|
+
syncDirectory(runDir);
|
|
564
|
+
}
|
|
565
|
+
|
|
261
566
|
function syncRootDirectory() {
|
|
262
567
|
const flags =
|
|
263
568
|
fs.constants.O_RDONLY |
|
|
@@ -328,7 +633,9 @@ function createPreviewPhaseOutbox(options) {
|
|
|
328
633
|
if (fd != null) {
|
|
329
634
|
try {
|
|
330
635
|
fileSystem.closeSync(fd);
|
|
331
|
-
} catch {
|
|
636
|
+
} catch {
|
|
637
|
+
// Best-effort fd cleanup.
|
|
638
|
+
}
|
|
332
639
|
}
|
|
333
640
|
try {
|
|
334
641
|
fileSystem.rmSync(tempPath, { force: true });
|
|
@@ -408,69 +715,21 @@ function createPreviewPhaseOutbox(options) {
|
|
|
408
715
|
|
|
409
716
|
function publishActiveProducerMarker() {
|
|
410
717
|
if (producerMarkerActive) return;
|
|
411
|
-
if (
|
|
718
|
+
if (
|
|
719
|
+
!producerRegistrationDurable ||
|
|
720
|
+
!metadata ||
|
|
721
|
+
!ownsSelectedWriterLock()
|
|
722
|
+
) {
|
|
412
723
|
throw new Error(
|
|
413
|
-
'preview phase producer marker requires
|
|
724
|
+
'preview phase producer marker requires authoritative registration'
|
|
414
725
|
);
|
|
415
726
|
}
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
throw producerMarkerConflict();
|
|
420
|
-
}
|
|
421
|
-
producerStartedAtMs = existing.started_at_ms;
|
|
422
|
-
producerMarkerActive = true;
|
|
423
|
-
return;
|
|
424
|
-
}
|
|
425
|
-
|
|
426
|
-
producerStartedAtMs = producerStartedAtMs ?? now();
|
|
427
|
-
const marker = buildProducerMarker('active');
|
|
428
|
-
const tempPath = path.join(
|
|
429
|
-
root,
|
|
430
|
-
`.producer-epoch.${producerEpoch}.${process.pid}.${writerInstanceID}.tmp`
|
|
431
|
-
);
|
|
432
|
-
let fd = null;
|
|
433
|
-
try {
|
|
434
|
-
const flags =
|
|
435
|
-
fs.constants.O_CREAT |
|
|
436
|
-
fs.constants.O_EXCL |
|
|
437
|
-
fs.constants.O_WRONLY |
|
|
438
|
-
(fs.constants.O_NOFOLLOW || 0);
|
|
439
|
-
fd = fileSystem.openSync(tempPath, flags, 0o600);
|
|
440
|
-
writeAll(fd, Buffer.from(`${JSON.stringify(marker)}\n`, 'utf8'));
|
|
441
|
-
fileSystem.fsyncSync(fd);
|
|
442
|
-
fileSystem.closeSync(fd);
|
|
443
|
-
fd = null;
|
|
444
|
-
try {
|
|
445
|
-
fileSystem.linkSync(tempPath, producerEpochMarkerPath);
|
|
446
|
-
syncRootDirectory();
|
|
447
|
-
} catch (error) {
|
|
448
|
-
if (error?.code !== 'EEXIST') throw error;
|
|
449
|
-
const existing = readAndValidateProducerMarker();
|
|
450
|
-
if (
|
|
451
|
-
!markerBelongsToThisWriter(existing) ||
|
|
452
|
-
existing.state !== 'active'
|
|
453
|
-
) {
|
|
454
|
-
throw producerMarkerConflict();
|
|
455
|
-
}
|
|
456
|
-
producerStartedAtMs = existing.started_at_ms;
|
|
457
|
-
}
|
|
458
|
-
producerMarkerActive = true;
|
|
459
|
-
} finally {
|
|
460
|
-
if (fd != null) {
|
|
461
|
-
try {
|
|
462
|
-
fileSystem.closeSync(fd);
|
|
463
|
-
} catch {}
|
|
464
|
-
}
|
|
465
|
-
let tempRemoved = false;
|
|
466
|
-
try {
|
|
467
|
-
fileSystem.rmSync(tempPath, { force: true });
|
|
468
|
-
tempRemoved = true;
|
|
469
|
-
} catch {
|
|
470
|
-
// A stale temp file is never accepted as lifecycle proof.
|
|
471
|
-
}
|
|
472
|
-
if (tempRemoved) syncRootDirectory();
|
|
727
|
+
const existing = readAndValidateProducerMarker();
|
|
728
|
+
if (!markerBelongsToThisWriter(existing) || existing.state !== 'active') {
|
|
729
|
+
throw producerMarkerConflict();
|
|
473
730
|
}
|
|
731
|
+
producerStartedAtMs = existing.started_at_ms;
|
|
732
|
+
producerMarkerActive = true;
|
|
474
733
|
}
|
|
475
734
|
|
|
476
735
|
function publishTerminalProducerMarker() {
|
|
@@ -513,7 +772,9 @@ function createPreviewPhaseOutbox(options) {
|
|
|
513
772
|
if (fd != null) {
|
|
514
773
|
try {
|
|
515
774
|
fileSystem.closeSync(fd);
|
|
516
|
-
} catch {
|
|
775
|
+
} catch {
|
|
776
|
+
// Best-effort fd cleanup.
|
|
777
|
+
}
|
|
517
778
|
}
|
|
518
779
|
let tempRemoved = false;
|
|
519
780
|
try {
|
|
@@ -526,13 +787,16 @@ function createPreviewPhaseOutbox(options) {
|
|
|
526
787
|
}
|
|
527
788
|
}
|
|
528
789
|
|
|
529
|
-
function
|
|
530
|
-
const
|
|
790
|
+
function readAndValidateMetadataAt(directoryPath) {
|
|
791
|
+
const candidateMetaPath = path.join(directoryPath, 'meta.json');
|
|
792
|
+
const stat = fileSystem.lstatSync(candidateMetaPath);
|
|
531
793
|
if (!stat.isFile() || stat.isSymbolicLink()) {
|
|
532
|
-
throw new Error(`unsafe preview phase outbox path: ${
|
|
794
|
+
throw new Error(`unsafe preview phase outbox path: ${candidateMetaPath}`);
|
|
533
795
|
}
|
|
534
|
-
fileSystem.chmodSync(
|
|
535
|
-
const metadata = JSON.parse(
|
|
796
|
+
fileSystem.chmodSync(candidateMetaPath, 0o600);
|
|
797
|
+
const metadata = JSON.parse(
|
|
798
|
+
fileSystem.readFileSync(candidateMetaPath, 'utf8')
|
|
799
|
+
);
|
|
536
800
|
if (
|
|
537
801
|
metadata.protocol !== 'preview-phase-outbox-v1' ||
|
|
538
802
|
metadata.schema_version !== 1 ||
|
|
@@ -544,11 +808,17 @@ function createPreviewPhaseOutbox(options) {
|
|
|
544
808
|
(metadata.quiesce_supported !== undefined &&
|
|
545
809
|
metadata.quiesce_supported !== true)
|
|
546
810
|
) {
|
|
547
|
-
throw new Error(
|
|
811
|
+
throw new Error(
|
|
812
|
+
`invalid preview phase outbox metadata: ${candidateMetaPath}`
|
|
813
|
+
);
|
|
548
814
|
}
|
|
549
815
|
return metadata;
|
|
550
816
|
}
|
|
551
817
|
|
|
818
|
+
function readAndValidateMetadata() {
|
|
819
|
+
return readAndValidateMetadataAt(runDir);
|
|
820
|
+
}
|
|
821
|
+
|
|
552
822
|
function ensureMetadata() {
|
|
553
823
|
if (metadata) return;
|
|
554
824
|
if (fileSystem.existsSync(metaPath)) {
|
|
@@ -580,15 +850,350 @@ function createPreviewPhaseOutbox(options) {
|
|
|
580
850
|
if (error?.code !== 'EEXIST') throw error;
|
|
581
851
|
}
|
|
582
852
|
} finally {
|
|
853
|
+
let removed = false;
|
|
583
854
|
try {
|
|
584
855
|
fileSystem.rmSync(tempPath, { force: true });
|
|
856
|
+
removed = true;
|
|
585
857
|
} catch {
|
|
586
858
|
// A stale temp file is harmless and never consumed by the reader.
|
|
587
859
|
}
|
|
860
|
+
if (removed) syncRunDirectory();
|
|
588
861
|
}
|
|
589
862
|
metadata = readAndValidateMetadata();
|
|
590
863
|
}
|
|
591
864
|
|
|
865
|
+
function validateFallbackIntent(intent, canonicalMetadata) {
|
|
866
|
+
if (
|
|
867
|
+
intent.protocol !== 'preview-phase-fallback-intent-v1' ||
|
|
868
|
+
intent.schema_version !== 1 ||
|
|
869
|
+
intent.producer_epoch !== producerEpoch ||
|
|
870
|
+
intent.run_id !== options.run_id ||
|
|
871
|
+
intent.generation !== canonicalMetadata.generation ||
|
|
872
|
+
!Number.isFinite(intent.created_at_ms) ||
|
|
873
|
+
intent.created_at_ms <= 0 ||
|
|
874
|
+
!STATE_ID_PATTERN.test(intent.reason || '')
|
|
875
|
+
) {
|
|
876
|
+
throw new Error(
|
|
877
|
+
`invalid preview phase fallback intent: ${path.join(
|
|
878
|
+
canonicalRunDir,
|
|
879
|
+
'fallback.intent.json'
|
|
880
|
+
)}`
|
|
881
|
+
);
|
|
882
|
+
}
|
|
883
|
+
return intent;
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
function fallbackSpoolError(code, message, cause) {
|
|
887
|
+
const error = new Error(message, cause == null ? undefined : { cause });
|
|
888
|
+
error.code = code;
|
|
889
|
+
return error;
|
|
890
|
+
}
|
|
891
|
+
|
|
892
|
+
function encodeFallbackRecord(record, description) {
|
|
893
|
+
const payload = JSON.stringify(record);
|
|
894
|
+
if (typeof payload !== 'string') {
|
|
895
|
+
throw fallbackSpoolError(
|
|
896
|
+
'PREVIEW_PHASE_FALLBACK_RECORD_INVALID',
|
|
897
|
+
`invalid preview phase fallback ${description}`
|
|
898
|
+
);
|
|
899
|
+
}
|
|
900
|
+
const encoded = Buffer.from(`${payload}\n`, 'utf8');
|
|
901
|
+
if (encoded.length > maxFallbackRecordBytes) {
|
|
902
|
+
throw fallbackSpoolError(
|
|
903
|
+
'PREVIEW_PHASE_FALLBACK_RECORD_TOO_LARGE',
|
|
904
|
+
`preview phase fallback ${description} exceeds ${maxFallbackRecordBytes} bytes`
|
|
905
|
+
);
|
|
906
|
+
}
|
|
907
|
+
return encoded;
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
function fallbackTimestamp() {
|
|
911
|
+
const timestamp = now();
|
|
912
|
+
if (!Number.isFinite(timestamp) || timestamp <= 0) {
|
|
913
|
+
throw fallbackSpoolError(
|
|
914
|
+
'PREVIEW_PHASE_FALLBACK_TIMESTAMP_INVALID',
|
|
915
|
+
'invalid preview phase fallback timestamp'
|
|
916
|
+
);
|
|
917
|
+
}
|
|
918
|
+
return timestamp;
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
function closeFallbackFD() {
|
|
922
|
+
if (fallbackFD == null) return;
|
|
923
|
+
try {
|
|
924
|
+
fileSystem.closeSync(fallbackFD);
|
|
925
|
+
} catch {
|
|
926
|
+
// The durable boundary is fdatasync, not close. Never reuse this fd.
|
|
927
|
+
}
|
|
928
|
+
fallbackFD = null;
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
function ensureFallbackSpool(initialEventBytes) {
|
|
932
|
+
if (fallbackFD != null) return;
|
|
933
|
+
if (fallbackSealed) {
|
|
934
|
+
throw fallbackSpoolError(
|
|
935
|
+
'PREVIEW_PHASE_FALLBACK_SEALED',
|
|
936
|
+
'preview phase fallback spool is sealed'
|
|
937
|
+
);
|
|
938
|
+
}
|
|
939
|
+
if (fallbackFailed) {
|
|
940
|
+
throw fallbackSpoolError(
|
|
941
|
+
'PREVIEW_PHASE_FALLBACK_DEGRADED',
|
|
942
|
+
'preview phase fallback spool is degraded'
|
|
943
|
+
);
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
assertSafeDirectory(fileSystem, root);
|
|
947
|
+
assertExistingSafeDirectory(fileSystem, canonicalRunDir);
|
|
948
|
+
const parentMetadata = readAndValidateMetadataAt(canonicalRunDir);
|
|
949
|
+
const entries = readDirectoryEntriesBounded(
|
|
950
|
+
canonicalRunDir,
|
|
951
|
+
maxRunScanEntries,
|
|
952
|
+
`preview phase run scan exceeds ${maxRunScanEntries} entries: ${canonicalRunDir}`
|
|
953
|
+
);
|
|
954
|
+
const fallbackFiles = entries.filter(entry =>
|
|
955
|
+
/^fallback-pfs_[a-f0-9]{32}\.jsonl$/.test(entry.name)
|
|
956
|
+
);
|
|
957
|
+
if (fallbackFiles.length >= maxFallbackFiles) {
|
|
958
|
+
throw fallbackSpoolError(
|
|
959
|
+
'PREVIEW_PHASE_FALLBACK_FILE_LIMIT',
|
|
960
|
+
`preview phase fallback file limit reached ${maxFallbackFiles}`
|
|
961
|
+
);
|
|
962
|
+
}
|
|
963
|
+
if (fileSystem.existsSync(fallbackSpoolPath)) {
|
|
964
|
+
throw fallbackSpoolError(
|
|
965
|
+
'PREVIEW_PHASE_FALLBACK_SESSION_CONFLICT',
|
|
966
|
+
`preview phase fallback session already exists: ${fallbackSessionID}`
|
|
967
|
+
);
|
|
968
|
+
}
|
|
969
|
+
|
|
970
|
+
const createdAt = fallbackTimestamp();
|
|
971
|
+
const metaRecord = encodeFallbackRecord(
|
|
972
|
+
{
|
|
973
|
+
protocol: 'preview-phase-fallback-spool-v1',
|
|
974
|
+
schema_version: 1,
|
|
975
|
+
record_type: 'meta',
|
|
976
|
+
session_id: fallbackSessionID,
|
|
977
|
+
producer_epoch: producerEpoch,
|
|
978
|
+
run_id: options.run_id,
|
|
979
|
+
parent_generation: parentMetadata.generation,
|
|
980
|
+
producer_instance_id: producerInstanceID,
|
|
981
|
+
pid: process.pid,
|
|
982
|
+
process_start_ticks: currentProcessStartTicks,
|
|
983
|
+
// Protocol timestamps use Unix epoch milliseconds. The compact field
|
|
984
|
+
// name is retained because this is a versioned wire contract.
|
|
985
|
+
created_at: createdAt,
|
|
986
|
+
},
|
|
987
|
+
'meta record'
|
|
988
|
+
);
|
|
989
|
+
if (
|
|
990
|
+
metaRecord.length + initialEventBytes + FALLBACK_TERMINAL_RESERVE_BYTES >
|
|
991
|
+
maxFallbackBytes
|
|
992
|
+
) {
|
|
993
|
+
throw fallbackSpoolError(
|
|
994
|
+
'PREVIEW_PHASE_FALLBACK_FILE_QUOTA_EXCEEDED',
|
|
995
|
+
`preview phase fallback file quota would exceed ${maxFallbackBytes} bytes`
|
|
996
|
+
);
|
|
997
|
+
}
|
|
998
|
+
assertRootByteQuota(
|
|
999
|
+
metaRecord.length + initialEventBytes + FALLBACK_TERMINAL_RESERVE_BYTES
|
|
1000
|
+
);
|
|
1001
|
+
|
|
1002
|
+
const tempPath = path.join(
|
|
1003
|
+
canonicalRunDir,
|
|
1004
|
+
`.fallback-${fallbackSessionID}.${process.pid}.${writerInstanceID}.tmp`
|
|
1005
|
+
);
|
|
1006
|
+
const directoryFlags =
|
|
1007
|
+
fs.constants.O_RDONLY |
|
|
1008
|
+
(fs.constants.O_DIRECTORY || 0) |
|
|
1009
|
+
(fs.constants.O_NOFOLLOW || 0);
|
|
1010
|
+
let directoryFD = null;
|
|
1011
|
+
let fd = null;
|
|
1012
|
+
let published = false;
|
|
1013
|
+
try {
|
|
1014
|
+
directoryFD = fileSystem.openSync(canonicalRunDir, directoryFlags);
|
|
1015
|
+
const fileFlags =
|
|
1016
|
+
fs.constants.O_CREAT |
|
|
1017
|
+
fs.constants.O_EXCL |
|
|
1018
|
+
fs.constants.O_WRONLY |
|
|
1019
|
+
fs.constants.O_APPEND |
|
|
1020
|
+
(fs.constants.O_NOFOLLOW || 0);
|
|
1021
|
+
fd = fileSystem.openSync(tempPath, fileFlags, 0o600);
|
|
1022
|
+
fileSystem.fchmodSync(fd, 0o600);
|
|
1023
|
+
writeAll(fd, metaRecord);
|
|
1024
|
+
fileSystem.fsyncSync(fd);
|
|
1025
|
+
try {
|
|
1026
|
+
fileSystem.linkSync(tempPath, fallbackSpoolPath);
|
|
1027
|
+
} catch (error) {
|
|
1028
|
+
if (error?.code === 'EEXIST') {
|
|
1029
|
+
throw fallbackSpoolError(
|
|
1030
|
+
'PREVIEW_PHASE_FALLBACK_SESSION_CONFLICT',
|
|
1031
|
+
`preview phase fallback session already exists: ${fallbackSessionID}`,
|
|
1032
|
+
error
|
|
1033
|
+
);
|
|
1034
|
+
}
|
|
1035
|
+
throw error;
|
|
1036
|
+
}
|
|
1037
|
+
// Sync the already-open directory, so quarantine rename after link does
|
|
1038
|
+
// not invalidate the durability proof for the published file name.
|
|
1039
|
+
fileSystem.fsyncSync(directoryFD);
|
|
1040
|
+
published = true;
|
|
1041
|
+
try {
|
|
1042
|
+
fileSystem.rmSync(tempPath);
|
|
1043
|
+
fileSystem.fsyncSync(directoryFD);
|
|
1044
|
+
} catch {
|
|
1045
|
+
// A quarantine rename can make the old path unreachable. The hidden
|
|
1046
|
+
// hard-link alias is harmless; the published session remains bounded.
|
|
1047
|
+
}
|
|
1048
|
+
fallbackFD = fd;
|
|
1049
|
+
fd = null;
|
|
1050
|
+
fallbackBytes = metaRecord.length;
|
|
1051
|
+
fallbackStarted = true;
|
|
1052
|
+
fallbackCreatedAt = createdAt;
|
|
1053
|
+
} catch (error) {
|
|
1054
|
+
fallbackFailed = true;
|
|
1055
|
+
throw error;
|
|
1056
|
+
} finally {
|
|
1057
|
+
if (fd != null) {
|
|
1058
|
+
try {
|
|
1059
|
+
fileSystem.closeSync(fd);
|
|
1060
|
+
} catch {
|
|
1061
|
+
// Best-effort fd cleanup.
|
|
1062
|
+
}
|
|
1063
|
+
}
|
|
1064
|
+
if (!published) {
|
|
1065
|
+
try {
|
|
1066
|
+
fileSystem.rmSync(tempPath, { force: true });
|
|
1067
|
+
if (directoryFD != null) fileSystem.fsyncSync(directoryFD);
|
|
1068
|
+
} catch {
|
|
1069
|
+
// Best-effort temp cleanup after a failed publication.
|
|
1070
|
+
}
|
|
1071
|
+
}
|
|
1072
|
+
if (directoryFD != null) {
|
|
1073
|
+
try {
|
|
1074
|
+
fileSystem.closeSync(directoryFD);
|
|
1075
|
+
} catch {
|
|
1076
|
+
// Best-effort directory-fd cleanup.
|
|
1077
|
+
}
|
|
1078
|
+
}
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
|
|
1082
|
+
function appendFallback(event) {
|
|
1083
|
+
if (fallbackSealed) {
|
|
1084
|
+
throw fallbackSpoolError(
|
|
1085
|
+
'PREVIEW_PHASE_FALLBACK_SEALED',
|
|
1086
|
+
'preview phase fallback spool is sealed'
|
|
1087
|
+
);
|
|
1088
|
+
}
|
|
1089
|
+
if (fallbackFailed) {
|
|
1090
|
+
throw fallbackSpoolError(
|
|
1091
|
+
'PREVIEW_PHASE_FALLBACK_DEGRADED',
|
|
1092
|
+
'preview phase fallback spool is degraded'
|
|
1093
|
+
);
|
|
1094
|
+
}
|
|
1095
|
+
if (!event || typeof event !== 'object' || Array.isArray(event)) {
|
|
1096
|
+
throw fallbackSpoolError(
|
|
1097
|
+
'PREVIEW_PHASE_FALLBACK_RECORD_INVALID',
|
|
1098
|
+
'invalid preview phase fallback event'
|
|
1099
|
+
);
|
|
1100
|
+
}
|
|
1101
|
+
if (fallbackSeq >= maxFallbackEvents) {
|
|
1102
|
+
throw fallbackSpoolError(
|
|
1103
|
+
'PREVIEW_PHASE_FALLBACK_EVENT_LIMIT',
|
|
1104
|
+
`preview phase fallback event limit reached ${maxFallbackEvents}`
|
|
1105
|
+
);
|
|
1106
|
+
}
|
|
1107
|
+
const nextSeq = fallbackSeq + 1;
|
|
1108
|
+
const fallbackEvent = {
|
|
1109
|
+
...event,
|
|
1110
|
+
transport: 'fallback_spool',
|
|
1111
|
+
fallback_session_id: fallbackSessionID,
|
|
1112
|
+
fallback_seq: nextSeq,
|
|
1113
|
+
};
|
|
1114
|
+
const eventPayload = JSON.stringify(fallbackEvent);
|
|
1115
|
+
if (
|
|
1116
|
+
typeof eventPayload !== 'string' ||
|
|
1117
|
+
Buffer.byteLength(eventPayload, 'utf8') > MAX_EVENT_BYTES
|
|
1118
|
+
) {
|
|
1119
|
+
throw fallbackSpoolError(
|
|
1120
|
+
'PREVIEW_PHASE_EVENT_TOO_LARGE',
|
|
1121
|
+
`preview phase event exceeds ${MAX_EVENT_BYTES} bytes`
|
|
1122
|
+
);
|
|
1123
|
+
}
|
|
1124
|
+
const record = encodeFallbackRecord(
|
|
1125
|
+
{
|
|
1126
|
+
protocol: 'preview-phase-fallback-spool-v1',
|
|
1127
|
+
schema_version: 1,
|
|
1128
|
+
record_type: 'event',
|
|
1129
|
+
session_id: fallbackSessionID,
|
|
1130
|
+
seq: nextSeq,
|
|
1131
|
+
event: fallbackEvent,
|
|
1132
|
+
},
|
|
1133
|
+
'event record'
|
|
1134
|
+
);
|
|
1135
|
+
|
|
1136
|
+
ensureFallbackSpool(record.length);
|
|
1137
|
+
if (
|
|
1138
|
+
fallbackBytes + record.length + FALLBACK_TERMINAL_RESERVE_BYTES >
|
|
1139
|
+
maxFallbackBytes
|
|
1140
|
+
) {
|
|
1141
|
+
throw fallbackSpoolError(
|
|
1142
|
+
'PREVIEW_PHASE_FALLBACK_FILE_QUOTA_EXCEEDED',
|
|
1143
|
+
`preview phase fallback file quota would exceed ${maxFallbackBytes} bytes`
|
|
1144
|
+
);
|
|
1145
|
+
}
|
|
1146
|
+
try {
|
|
1147
|
+
writeAll(fallbackFD, record);
|
|
1148
|
+
fileSystem.fdatasyncSync(fallbackFD);
|
|
1149
|
+
fallbackBytes += record.length;
|
|
1150
|
+
fallbackSeq = nextSeq;
|
|
1151
|
+
fallbackRequired = true;
|
|
1152
|
+
return true;
|
|
1153
|
+
} catch (error) {
|
|
1154
|
+
fallbackFailed = true;
|
|
1155
|
+
closeFallbackFD();
|
|
1156
|
+
throw error;
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
|
|
1160
|
+
function sealFallback() {
|
|
1161
|
+
if (!fallbackStarted) return !fallbackFailed;
|
|
1162
|
+
if (fallbackSealed) return true;
|
|
1163
|
+
if (fallbackFailed || fallbackFD == null) return false;
|
|
1164
|
+
const terminal = encodeFallbackRecord(
|
|
1165
|
+
{
|
|
1166
|
+
protocol: 'preview-phase-fallback-spool-v1',
|
|
1167
|
+
schema_version: 1,
|
|
1168
|
+
record_type: 'terminal',
|
|
1169
|
+
session_id: fallbackSessionID,
|
|
1170
|
+
final_cursor: fallbackBytes,
|
|
1171
|
+
final_seq: fallbackSeq,
|
|
1172
|
+
// Unix epoch milliseconds; never precedes this session's meta record.
|
|
1173
|
+
sealed_at: Math.max(fallbackTimestamp(), fallbackCreatedAt),
|
|
1174
|
+
recovered: false,
|
|
1175
|
+
},
|
|
1176
|
+
'terminal record'
|
|
1177
|
+
);
|
|
1178
|
+
if (fallbackBytes + terminal.length > maxFallbackBytes) {
|
|
1179
|
+
fallbackFailed = true;
|
|
1180
|
+
closeFallbackFD();
|
|
1181
|
+
return false;
|
|
1182
|
+
}
|
|
1183
|
+
try {
|
|
1184
|
+
writeAll(fallbackFD, terminal);
|
|
1185
|
+
fileSystem.fdatasyncSync(fallbackFD);
|
|
1186
|
+
fallbackBytes += terminal.length;
|
|
1187
|
+
fallbackSealed = true;
|
|
1188
|
+
closeFallbackFD();
|
|
1189
|
+
return true;
|
|
1190
|
+
} catch {
|
|
1191
|
+
fallbackFailed = true;
|
|
1192
|
+
closeFallbackFD();
|
|
1193
|
+
return false;
|
|
1194
|
+
}
|
|
1195
|
+
}
|
|
1196
|
+
|
|
592
1197
|
function validateProcessIdentityState(state, description) {
|
|
593
1198
|
if (
|
|
594
1199
|
!Number.isSafeInteger(state.pid) ||
|
|
@@ -623,8 +1228,22 @@ function createPreviewPhaseOutbox(options) {
|
|
|
623
1228
|
return busy;
|
|
624
1229
|
}
|
|
625
1230
|
|
|
1231
|
+
function runConflictError(cause) {
|
|
1232
|
+
const conflict = new Error(
|
|
1233
|
+
`preview phase canonical run was published by another writer: ${options.run_id}`,
|
|
1234
|
+
{ cause }
|
|
1235
|
+
);
|
|
1236
|
+
conflict.code = 'PREVIEW_PHASE_RUN_CONFLICT';
|
|
1237
|
+
return conflict;
|
|
1238
|
+
}
|
|
1239
|
+
|
|
626
1240
|
function acquireWriterLock() {
|
|
627
|
-
if (
|
|
1241
|
+
if (ownsSelectedWriterLock()) return;
|
|
1242
|
+
if (writerLockOwned) {
|
|
1243
|
+
throw new Error(
|
|
1244
|
+
'preview phase writer ownership belongs to another directory'
|
|
1245
|
+
);
|
|
1246
|
+
}
|
|
628
1247
|
const lock = {
|
|
629
1248
|
protocol: 'preview-phase-writer-v1',
|
|
630
1249
|
schema_version: 1,
|
|
@@ -649,6 +1268,8 @@ function createPreviewPhaseOutbox(options) {
|
|
|
649
1268
|
try {
|
|
650
1269
|
fileSystem.linkSync(tempPath, writerLockPath);
|
|
651
1270
|
writerLockOwned = true;
|
|
1271
|
+
writerLockDirectory = runDir;
|
|
1272
|
+
if (runDir === canonicalRunDir) everOwnedCanonicalWriter = true;
|
|
652
1273
|
syncRunDirectory();
|
|
653
1274
|
} catch (error) {
|
|
654
1275
|
if (error?.code !== 'EEXIST') throw error;
|
|
@@ -659,12 +1280,234 @@ function createPreviewPhaseOutbox(options) {
|
|
|
659
1280
|
throw writerBusyError();
|
|
660
1281
|
}
|
|
661
1282
|
} finally {
|
|
1283
|
+
let removed = false;
|
|
662
1284
|
try {
|
|
663
1285
|
fileSystem.rmSync(tempPath, { force: true });
|
|
1286
|
+
removed = true;
|
|
664
1287
|
} catch {
|
|
665
1288
|
// A stale temp file is harmless and never consumed by the reader.
|
|
666
1289
|
}
|
|
1290
|
+
if (removed) syncRunDirectory();
|
|
1291
|
+
}
|
|
1292
|
+
}
|
|
1293
|
+
|
|
1294
|
+
function ensureEmptyEventsFile() {
|
|
1295
|
+
if (fileSystem.existsSync(eventsPath)) {
|
|
1296
|
+
const stat = fileSystem.lstatSync(eventsPath);
|
|
1297
|
+
if (!stat.isFile() || stat.isSymbolicLink()) {
|
|
1298
|
+
throw new Error(`unsafe preview phase outbox path: ${eventsPath}`);
|
|
1299
|
+
}
|
|
1300
|
+
return;
|
|
1301
|
+
}
|
|
1302
|
+
const flags =
|
|
1303
|
+
fs.constants.O_CREAT |
|
|
1304
|
+
fs.constants.O_EXCL |
|
|
1305
|
+
fs.constants.O_RDWR |
|
|
1306
|
+
(fs.constants.O_NOFOLLOW || 0);
|
|
1307
|
+
const fd = fileSystem.openSync(eventsPath, flags, 0o600);
|
|
1308
|
+
try {
|
|
1309
|
+
fileSystem.fchmodSync(fd, 0o600);
|
|
1310
|
+
fileSystem.fdatasyncSync(fd);
|
|
1311
|
+
} finally {
|
|
1312
|
+
fileSystem.closeSync(fd);
|
|
1313
|
+
}
|
|
1314
|
+
syncRunDirectory();
|
|
1315
|
+
}
|
|
1316
|
+
|
|
1317
|
+
function validateOwnedWriterLock() {
|
|
1318
|
+
const lock = validateWriterLock(
|
|
1319
|
+
readRegularJSON(writerLockPath, 'writer lock')
|
|
1320
|
+
);
|
|
1321
|
+
if (
|
|
1322
|
+
lock.writer_instance_id !== writerInstanceID ||
|
|
1323
|
+
lock.pid !== process.pid ||
|
|
1324
|
+
(lock.process_start_ticks || '') !== (currentProcessStartTicks || '')
|
|
1325
|
+
) {
|
|
1326
|
+
throw writerBusyError();
|
|
1327
|
+
}
|
|
1328
|
+
writerLockOwned = true;
|
|
1329
|
+
writerLockDirectory = runDir;
|
|
1330
|
+
}
|
|
1331
|
+
|
|
1332
|
+
let stagingOwned = false;
|
|
1333
|
+
|
|
1334
|
+
function tryAdoptCanonicalWriter() {
|
|
1335
|
+
const previousDirectory = runDir;
|
|
1336
|
+
const previousWriterLockOwned = writerLockOwned;
|
|
1337
|
+
const previousWriterLockDirectory = writerLockDirectory;
|
|
1338
|
+
try {
|
|
1339
|
+
selectRunDirectory(canonicalRunDir);
|
|
1340
|
+
assertExistingSafeDirectory(fileSystem, canonicalRunDir);
|
|
1341
|
+
metadata = readAndValidateMetadata();
|
|
1342
|
+
writerLockOwned = false;
|
|
1343
|
+
writerLockDirectory = null;
|
|
1344
|
+
validateOwnedWriterLock();
|
|
1345
|
+
stagingOwned = false;
|
|
1346
|
+
everOwnedCanonicalWriter = true;
|
|
1347
|
+
return true;
|
|
1348
|
+
} catch {
|
|
1349
|
+
selectRunDirectory(previousDirectory);
|
|
1350
|
+
writerLockOwned = previousWriterLockOwned;
|
|
1351
|
+
writerLockDirectory = previousWriterLockDirectory;
|
|
1352
|
+
return false;
|
|
1353
|
+
}
|
|
1354
|
+
}
|
|
1355
|
+
|
|
1356
|
+
function ensureCanonicalPublicationDurable() {
|
|
1357
|
+
if (canonicalPublicationDurable) return;
|
|
1358
|
+
try {
|
|
1359
|
+
syncRootDirectory();
|
|
1360
|
+
} catch (firstError) {
|
|
1361
|
+
// rename may have committed even when its first parent-directory fsync
|
|
1362
|
+
// failed. Re-prove the exact writer identity, then retry that commit
|
|
1363
|
+
// point before any event or producer marker can be published.
|
|
1364
|
+
if (!tryAdoptCanonicalWriter()) throw firstError;
|
|
1365
|
+
syncRootDirectory();
|
|
1366
|
+
}
|
|
1367
|
+
canonicalPublicationDurable = true;
|
|
1368
|
+
}
|
|
1369
|
+
|
|
1370
|
+
function registerCanonicalWriter(rejectedEvent) {
|
|
1371
|
+
if (producerRegistrationDurable) return;
|
|
1372
|
+
producerStartedAtMs = producerStartedAtMs ?? now();
|
|
1373
|
+
const rejectedEventForRegistration =
|
|
1374
|
+
rejectedEvent?.schema_version === 1 &&
|
|
1375
|
+
typeof rejectedEvent.event_id === 'string' &&
|
|
1376
|
+
rejectedEvent.event_id.startsWith(`${options.run_id}:`) &&
|
|
1377
|
+
rejectedEvent.run_id === options.run_id &&
|
|
1378
|
+
PRODUCER_INSTANCE_ID_PATTERN.test(
|
|
1379
|
+
rejectedEvent.producer_instance_id || ''
|
|
1380
|
+
) &&
|
|
1381
|
+
rejectedEvent.producer_seq === 1 &&
|
|
1382
|
+
rejectedEvent.transport === 'outbox'
|
|
1383
|
+
? rejectedEvent
|
|
1384
|
+
: null;
|
|
1385
|
+
const request = {
|
|
1386
|
+
// root/staging_name/started_at_ms are local callback context only. The
|
|
1387
|
+
// default registrar serializes the fixed protocol whitelist below and
|
|
1388
|
+
// never sends a caller-controlled path to shell-server.
|
|
1389
|
+
root,
|
|
1390
|
+
staging_name: stagingName,
|
|
1391
|
+
run_id: options.run_id,
|
|
1392
|
+
generation: metadata.generation,
|
|
1393
|
+
expected_instance_id: expectedInstanceID,
|
|
1394
|
+
producer_epoch: producerEpoch,
|
|
1395
|
+
writer_instance_id: writerInstanceID,
|
|
1396
|
+
pid: process.pid,
|
|
1397
|
+
process_start_ticks: currentProcessStartTicks,
|
|
1398
|
+
started_at_ms: producerStartedAtMs,
|
|
1399
|
+
...(rejectedEventForRegistration == null
|
|
1400
|
+
? {}
|
|
1401
|
+
: {
|
|
1402
|
+
producer_instance_id:
|
|
1403
|
+
rejectedEventForRegistration.producer_instance_id,
|
|
1404
|
+
fallback_session_id: fallbackSessionID,
|
|
1405
|
+
rejected_event: rejectedEventForRegistration,
|
|
1406
|
+
}),
|
|
1407
|
+
};
|
|
1408
|
+
if (producerRegistrationAttemptCount >= PRODUCER_REGISTER_ATTEMPTS) {
|
|
1409
|
+
throw producerRegistrationLastError;
|
|
1410
|
+
}
|
|
1411
|
+
while (producerRegistrationAttemptCount < PRODUCER_REGISTER_ATTEMPTS) {
|
|
1412
|
+
producerRegistrationAttemptCount += 1;
|
|
1413
|
+
try {
|
|
1414
|
+
const response = validateProducerRegistrationResponse(
|
|
1415
|
+
registerProducer(request),
|
|
1416
|
+
request
|
|
1417
|
+
);
|
|
1418
|
+
if (response.registration_status === 'rejected') {
|
|
1419
|
+
request.rejected_event.transport = 'fallback_spool';
|
|
1420
|
+
request.rejected_event.fallback_session_id = fallbackSessionID;
|
|
1421
|
+
request.rejected_event.fallback_seq = 1;
|
|
1422
|
+
fallbackRequired = true;
|
|
1423
|
+
fallbackSeq = 1;
|
|
1424
|
+
fallbackSealed = true;
|
|
1425
|
+
sealRequested = true;
|
|
1426
|
+
closed = true;
|
|
1427
|
+
closeResult = true;
|
|
1428
|
+
writerLockOwned = false;
|
|
1429
|
+
writerLockDirectory = null;
|
|
1430
|
+
stagingOwned = false;
|
|
1431
|
+
producerRegistrationDurable = true;
|
|
1432
|
+
selectRunDirectory(canonicalRunDir);
|
|
1433
|
+
return;
|
|
1434
|
+
}
|
|
1435
|
+
if (!tryAdoptCanonicalWriter()) {
|
|
1436
|
+
throw producerRegistrationError(
|
|
1437
|
+
'PREVIEW_PHASE_PRODUCER_REGISTRATION_ACK_UNKNOWN',
|
|
1438
|
+
'authoritative registration did not publish the expected canonical writer'
|
|
1439
|
+
);
|
|
1440
|
+
}
|
|
1441
|
+
// A successful authoritative response is issued only after the
|
|
1442
|
+
// canonical rename and root-directory fsync. Re-fsyncing here would
|
|
1443
|
+
// add latency without strengthening the controller's commit proof.
|
|
1444
|
+
canonicalPublicationDurable = true;
|
|
1445
|
+
const marker = readAndValidateProducerMarker();
|
|
1446
|
+
if (!markerBelongsToThisWriter(marker) || marker.state !== 'active') {
|
|
1447
|
+
throw producerRegistrationError(
|
|
1448
|
+
'PREVIEW_PHASE_PRODUCER_REGISTRATION_ACK_UNKNOWN',
|
|
1449
|
+
'authoritative registration did not publish the expected active marker'
|
|
1450
|
+
);
|
|
1451
|
+
}
|
|
1452
|
+
producerStartedAtMs = marker.started_at_ms;
|
|
1453
|
+
producerMarkerActive = true;
|
|
1454
|
+
producerRegistrationDurable = true;
|
|
1455
|
+
return;
|
|
1456
|
+
} catch (error) {
|
|
1457
|
+
producerRegistrationLastError = error;
|
|
1458
|
+
if (
|
|
1459
|
+
error?.code === 'PREVIEW_PHASE_PRODUCER_EPOCH_CONFLICT' &&
|
|
1460
|
+
tryAdoptCanonicalWriter()
|
|
1461
|
+
) {
|
|
1462
|
+
canonicalPublicationDurable = true;
|
|
1463
|
+
}
|
|
1464
|
+
if (
|
|
1465
|
+
error?.code === 'PREVIEW_PHASE_PRODUCER_REGISTRATION_ACK_UNKNOWN' &&
|
|
1466
|
+
producerRegistrationAttemptCount < PRODUCER_REGISTER_ATTEMPTS
|
|
1467
|
+
) {
|
|
1468
|
+
continue;
|
|
1469
|
+
}
|
|
1470
|
+
throw error;
|
|
1471
|
+
}
|
|
1472
|
+
}
|
|
1473
|
+
throw producerRegistrationLastError;
|
|
1474
|
+
}
|
|
1475
|
+
|
|
1476
|
+
function ensureCanonicalWriter(rejectedEvent) {
|
|
1477
|
+
if (producerRegistrationDurable && runDir === canonicalRunDir && metadata) {
|
|
1478
|
+
ensureCanonicalPublicationDurable();
|
|
1479
|
+
return;
|
|
1480
|
+
}
|
|
1481
|
+
|
|
1482
|
+
if (!stagingOwned && fileSystem.existsSync(canonicalRunDir)) {
|
|
1483
|
+
// Existing canonical state is controller-owned. A JS successor may
|
|
1484
|
+
// inspect it to produce a precise fail-closed reason, but it must never
|
|
1485
|
+
// acquire/remove its writer lock or republish it under a captured epoch.
|
|
1486
|
+
selectRunDirectory(canonicalRunDir);
|
|
1487
|
+
assertExistingSafeDirectory(fileSystem, canonicalRunDir);
|
|
1488
|
+
metadata = readAndValidateMetadata();
|
|
1489
|
+
if (fileSystem.existsSync(sealPath)) {
|
|
1490
|
+
assertGenerationIsOpen();
|
|
1491
|
+
}
|
|
1492
|
+
if (fileSystem.existsSync(writerLockPath)) {
|
|
1493
|
+
validateWriterLock(readRegularJSON(writerLockPath, 'writer lock'));
|
|
1494
|
+
throw writerBusyError();
|
|
1495
|
+
}
|
|
1496
|
+
throw runConflictError();
|
|
667
1497
|
}
|
|
1498
|
+
|
|
1499
|
+
selectRunDirectory(stagingRunDir);
|
|
1500
|
+
if (!stagingOwned) {
|
|
1501
|
+
fileSystem.mkdirSync(stagingRunDir, { mode: 0o700 });
|
|
1502
|
+
stagingOwned = true;
|
|
1503
|
+
syncRootDirectory();
|
|
1504
|
+
}
|
|
1505
|
+
assertSafeDirectory(fileSystem, stagingRunDir);
|
|
1506
|
+
ensureMetadata();
|
|
1507
|
+
if (!ownsSelectedWriterLock()) acquireWriterLock();
|
|
1508
|
+
ensureEmptyEventsFile();
|
|
1509
|
+
syncRunDirectory();
|
|
1510
|
+
registerCanonicalWriter(rejectedEvent);
|
|
668
1511
|
}
|
|
669
1512
|
|
|
670
1513
|
function assertStateGeneration(state, description) {
|
|
@@ -682,7 +1525,7 @@ function createPreviewPhaseOutbox(options) {
|
|
|
682
1525
|
|
|
683
1526
|
function assertGenerationIsOpen() {
|
|
684
1527
|
if (!fileSystem.existsSync(sealPath)) return;
|
|
685
|
-
|
|
1528
|
+
readAndValidateSeal();
|
|
686
1529
|
const error = new Error(
|
|
687
1530
|
`preview phase sealed generation cannot be reopened: ${options.run_id}`
|
|
688
1531
|
);
|
|
@@ -727,6 +1570,16 @@ function createPreviewPhaseOutbox(options) {
|
|
|
727
1570
|
return true;
|
|
728
1571
|
}
|
|
729
1572
|
|
|
1573
|
+
function observeFallbackIntent() {
|
|
1574
|
+
if (!fileSystem.existsSync(fallbackIntentPath)) return false;
|
|
1575
|
+
validateFallbackIntent(
|
|
1576
|
+
readRegularJSON(fallbackIntentPath, 'fallback intent'),
|
|
1577
|
+
metadata
|
|
1578
|
+
);
|
|
1579
|
+
fallbackRequired = true;
|
|
1580
|
+
return true;
|
|
1581
|
+
}
|
|
1582
|
+
|
|
730
1583
|
function assertGenerationIsNotDegraded() {
|
|
731
1584
|
if (!observeDegradedMarker()) return;
|
|
732
1585
|
throw degradedError();
|
|
@@ -745,6 +1598,7 @@ function createPreviewPhaseOutbox(options) {
|
|
|
745
1598
|
}
|
|
746
1599
|
fileSystem.rmSync(writerLockPath);
|
|
747
1600
|
writerLockOwned = false;
|
|
1601
|
+
writerLockDirectory = null;
|
|
748
1602
|
syncRunDirectory();
|
|
749
1603
|
}
|
|
750
1604
|
|
|
@@ -800,7 +1654,7 @@ function createPreviewPhaseOutbox(options) {
|
|
|
800
1654
|
fallbackRequired = true;
|
|
801
1655
|
if (degraded) return;
|
|
802
1656
|
degraded = true;
|
|
803
|
-
if (!metadata) return;
|
|
1657
|
+
if (!metadata || !ownsSelectedWriterLock()) return;
|
|
804
1658
|
const reason =
|
|
805
1659
|
typeof cause?.code === 'string' && STATE_ID_PATTERN.test(cause.code)
|
|
806
1660
|
? cause.code
|
|
@@ -812,7 +1666,9 @@ function createPreviewPhaseOutbox(options) {
|
|
|
812
1666
|
run_id: options.run_id,
|
|
813
1667
|
generation: metadata.generation,
|
|
814
1668
|
first_failed_event_id:
|
|
815
|
-
typeof event?.event_id === 'string'
|
|
1669
|
+
typeof event?.event_id === 'string'
|
|
1670
|
+
? event.event_id
|
|
1671
|
+
: `${options.run_id}:transport-fallback`,
|
|
816
1672
|
degraded_at_ms: now(),
|
|
817
1673
|
reason,
|
|
818
1674
|
});
|
|
@@ -822,6 +1678,153 @@ function createPreviewPhaseOutbox(options) {
|
|
|
822
1678
|
}
|
|
823
1679
|
}
|
|
824
1680
|
|
|
1681
|
+
function finalizeUnboundFallback(event, conflict) {
|
|
1682
|
+
fallbackRequired = true;
|
|
1683
|
+
latchDegraded(event, conflict);
|
|
1684
|
+
recoverTornTail();
|
|
1685
|
+
ensureEmptyEventsFile();
|
|
1686
|
+
const fd = fileSystem.openSync(
|
|
1687
|
+
eventsPath,
|
|
1688
|
+
fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0)
|
|
1689
|
+
);
|
|
1690
|
+
let finalSize;
|
|
1691
|
+
try {
|
|
1692
|
+
finalSize = fileSystem.fstatSync(fd).size;
|
|
1693
|
+
} finally {
|
|
1694
|
+
fileSystem.closeSync(fd);
|
|
1695
|
+
}
|
|
1696
|
+
const ownedWriter = validateWriterLock(
|
|
1697
|
+
readRegularJSON(writerLockPath, 'writer lock')
|
|
1698
|
+
);
|
|
1699
|
+
if (
|
|
1700
|
+
ownedWriter.writer_instance_id !== writerInstanceID ||
|
|
1701
|
+
ownedWriter.pid !== process.pid ||
|
|
1702
|
+
(ownedWriter.process_start_ticks || '') !==
|
|
1703
|
+
(currentProcessStartTicks || '')
|
|
1704
|
+
) {
|
|
1705
|
+
throw writerBusyError();
|
|
1706
|
+
}
|
|
1707
|
+
const terminalAt = Math.max(now(), ownedWriter.created_at_ms);
|
|
1708
|
+
if (!fileSystem.existsSync(sealPath)) {
|
|
1709
|
+
publishImmutableState(sealPath, 'sealed', {
|
|
1710
|
+
protocol: 'preview-phase-seal-v1',
|
|
1711
|
+
schema_version: 1,
|
|
1712
|
+
run_id: options.run_id,
|
|
1713
|
+
generation: metadata.generation,
|
|
1714
|
+
final_size_bytes: finalSize,
|
|
1715
|
+
sealed_at_ms: terminalAt,
|
|
1716
|
+
fallback_required: true,
|
|
1717
|
+
});
|
|
1718
|
+
}
|
|
1719
|
+
const observedOwner = readAndValidateProducerMarker();
|
|
1720
|
+
if (!fileSystem.existsSync(unboundFallbackPath)) {
|
|
1721
|
+
publishImmutableState(unboundFallbackPath, 'producer-unbound', {
|
|
1722
|
+
protocol: 'preview-phase-unbound-fallback-v1',
|
|
1723
|
+
schema_version: 1,
|
|
1724
|
+
epoch: producerEpoch,
|
|
1725
|
+
run_id: options.run_id,
|
|
1726
|
+
generation: metadata.generation,
|
|
1727
|
+
writer_instance_id: writerInstanceID,
|
|
1728
|
+
pid: process.pid,
|
|
1729
|
+
process_start_ticks: currentProcessStartTicks,
|
|
1730
|
+
started_at_ms: ownedWriter.created_at_ms,
|
|
1731
|
+
state: 'terminal',
|
|
1732
|
+
terminal_at_ms: terminalAt,
|
|
1733
|
+
observed_owner_run_id: observedOwner.run_id,
|
|
1734
|
+
observed_owner_generation: observedOwner.generation,
|
|
1735
|
+
});
|
|
1736
|
+
}
|
|
1737
|
+
if (ownsSelectedWriterLock()) removeOwnedWriterLock();
|
|
1738
|
+
closed = true;
|
|
1739
|
+
closeResult = true;
|
|
1740
|
+
}
|
|
1741
|
+
|
|
1742
|
+
function finalizeUnpublishedStagingFallback(event, conflict) {
|
|
1743
|
+
if (
|
|
1744
|
+
runDir !== stagingRunDir ||
|
|
1745
|
+
!stagingOwned ||
|
|
1746
|
+
!ownsSelectedWriterLock()
|
|
1747
|
+
) {
|
|
1748
|
+
return false;
|
|
1749
|
+
}
|
|
1750
|
+
ensureMetadata();
|
|
1751
|
+
fallbackRequired = true;
|
|
1752
|
+
latchDegraded(event, conflict);
|
|
1753
|
+
recoverTornTail();
|
|
1754
|
+
ensureEmptyEventsFile();
|
|
1755
|
+
const fd = fileSystem.openSync(
|
|
1756
|
+
eventsPath,
|
|
1757
|
+
fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0)
|
|
1758
|
+
);
|
|
1759
|
+
let finalSize;
|
|
1760
|
+
try {
|
|
1761
|
+
finalSize = fileSystem.fstatSync(fd).size;
|
|
1762
|
+
} finally {
|
|
1763
|
+
fileSystem.closeSync(fd);
|
|
1764
|
+
}
|
|
1765
|
+
const ownedWriter = validateWriterLock(
|
|
1766
|
+
readRegularJSON(writerLockPath, 'writer lock')
|
|
1767
|
+
);
|
|
1768
|
+
if (
|
|
1769
|
+
ownedWriter.writer_instance_id !== writerInstanceID ||
|
|
1770
|
+
ownedWriter.pid !== process.pid ||
|
|
1771
|
+
(ownedWriter.process_start_ticks || '') !==
|
|
1772
|
+
(currentProcessStartTicks || '')
|
|
1773
|
+
) {
|
|
1774
|
+
throw writerBusyError();
|
|
1775
|
+
}
|
|
1776
|
+
if (!fileSystem.existsSync(sealPath)) {
|
|
1777
|
+
publishImmutableState(sealPath, 'sealed', {
|
|
1778
|
+
protocol: 'preview-phase-seal-v1',
|
|
1779
|
+
schema_version: 1,
|
|
1780
|
+
run_id: options.run_id,
|
|
1781
|
+
generation: metadata.generation,
|
|
1782
|
+
final_size_bytes: finalSize,
|
|
1783
|
+
sealed_at_ms: Math.max(now(), ownedWriter.created_at_ms),
|
|
1784
|
+
fallback_required: true,
|
|
1785
|
+
});
|
|
1786
|
+
}
|
|
1787
|
+
removeOwnedWriterLock();
|
|
1788
|
+
closed = true;
|
|
1789
|
+
closeResult = true;
|
|
1790
|
+
try {
|
|
1791
|
+
const allowed = new Set([
|
|
1792
|
+
'meta.json',
|
|
1793
|
+
'events.jsonl',
|
|
1794
|
+
'degraded.json',
|
|
1795
|
+
'sealed.json',
|
|
1796
|
+
]);
|
|
1797
|
+
const entries = readDirectoryEntriesBounded(
|
|
1798
|
+
stagingRunDir,
|
|
1799
|
+
allowed.size,
|
|
1800
|
+
'preview phase unpublished staging exceeds cleanup entry limit'
|
|
1801
|
+
);
|
|
1802
|
+
for (const entry of entries) {
|
|
1803
|
+
if (!allowed.has(entry.name)) {
|
|
1804
|
+
throw new Error(
|
|
1805
|
+
'preview phase unpublished staging contains unknown state'
|
|
1806
|
+
);
|
|
1807
|
+
}
|
|
1808
|
+
const entryPath = path.join(stagingRunDir, entry.name);
|
|
1809
|
+
const stat = fileSystem.lstatSync(entryPath);
|
|
1810
|
+
if (!stat.isFile() || stat.isSymbolicLink()) {
|
|
1811
|
+
throw new Error(
|
|
1812
|
+
'preview phase unpublished staging contains unsafe state'
|
|
1813
|
+
);
|
|
1814
|
+
}
|
|
1815
|
+
fileSystem.rmSync(entryPath);
|
|
1816
|
+
}
|
|
1817
|
+
syncRunDirectory();
|
|
1818
|
+
fileSystem.rmdirSync(stagingRunDir);
|
|
1819
|
+
syncRootDirectory();
|
|
1820
|
+
stagingOwned = false;
|
|
1821
|
+
} catch {
|
|
1822
|
+
// The writer lock is already durably absent. Trusted controller GC can
|
|
1823
|
+
// finish an unexpected cleanup remainder without touching canonical.
|
|
1824
|
+
}
|
|
1825
|
+
return true;
|
|
1826
|
+
}
|
|
1827
|
+
|
|
825
1828
|
function append(event) {
|
|
826
1829
|
if (closed) throw new Error('preview phase outbox is closed');
|
|
827
1830
|
if (degraded) throw degradedError();
|
|
@@ -834,89 +1837,144 @@ function createPreviewPhaseOutbox(options) {
|
|
|
834
1837
|
throw error;
|
|
835
1838
|
}
|
|
836
1839
|
const record = Buffer.from(`${payload}\n`, 'utf8');
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
if
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
1840
|
+
for (let attempt = 1; attempt <= MAX_TRANSIENT_ATTEMPTS; attempt += 1) {
|
|
1841
|
+
try {
|
|
1842
|
+
assertSafeDirectory(fileSystem, root);
|
|
1843
|
+
assertRootDirectoryQuota();
|
|
1844
|
+
ensureCanonicalWriter(event);
|
|
1845
|
+
if (
|
|
1846
|
+
closed &&
|
|
1847
|
+
producerRegistrationDurable &&
|
|
1848
|
+
!everOwnedCanonicalWriter
|
|
1849
|
+
) {
|
|
1850
|
+
return true;
|
|
1851
|
+
}
|
|
1852
|
+
// The authoritative registrar proves the initial writer identity, but
|
|
1853
|
+
// every append must still fail closed if a controller or an unexpected
|
|
1854
|
+
// same-UID actor replaces auxiliary state after registration.
|
|
1855
|
+
validateOwnedWriterLock();
|
|
1856
|
+
assertGenerationIsOpen();
|
|
1857
|
+
publishActiveProducerMarker();
|
|
1858
|
+
// Any existing writer identity is authoritative to JS. Dead-owner
|
|
1859
|
+
// recovery and torn-tail finalization belong exclusively to shell-server.
|
|
1860
|
+
assertGenerationIsNotDegraded();
|
|
1861
|
+
assertRootByteQuota(record.length);
|
|
1862
|
+
if (eventsFD == null) {
|
|
1863
|
+
if (fileSystem.existsSync(eventsPath)) {
|
|
1864
|
+
const stat = fileSystem.lstatSync(eventsPath);
|
|
1865
|
+
if (!stat.isFile() || stat.isSymbolicLink()) {
|
|
1866
|
+
throw new Error(
|
|
1867
|
+
`unsafe preview phase outbox path: ${eventsPath}`
|
|
1868
|
+
);
|
|
1869
|
+
}
|
|
854
1870
|
}
|
|
1871
|
+
recoverTornTail();
|
|
1872
|
+
const flags =
|
|
1873
|
+
fs.constants.O_CREAT |
|
|
1874
|
+
fs.constants.O_WRONLY |
|
|
1875
|
+
fs.constants.O_APPEND |
|
|
1876
|
+
(fs.constants.O_NOFOLLOW || 0);
|
|
1877
|
+
eventsFD = fileSystem.openSync(eventsPath, flags, 0o600);
|
|
1878
|
+
fileSystem.fchmodSync(eventsFD, 0o600);
|
|
1879
|
+
committedBytes = fileSystem.fstatSync(eventsFD).size;
|
|
855
1880
|
}
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
eventsFD = fileSystem.openSync(eventsPath, flags, 0o600);
|
|
863
|
-
fileSystem.fchmodSync(eventsFD, 0o600);
|
|
864
|
-
committedBytes = fileSystem.fstatSync(eventsFD).size;
|
|
865
|
-
}
|
|
866
|
-
if (committedBytes + record.length > maxRunBytes) {
|
|
867
|
-
const error = new Error(
|
|
868
|
-
`preview phase run quota would exceed ${maxRunBytes} bytes`
|
|
869
|
-
);
|
|
870
|
-
error.code = 'PREVIEW_PHASE_RUN_QUOTA_EXCEEDED';
|
|
871
|
-
throw error;
|
|
872
|
-
}
|
|
873
|
-
let offset = 0;
|
|
874
|
-
while (offset < record.length) {
|
|
875
|
-
const written = fileSystem.writeSync(
|
|
876
|
-
eventsFD,
|
|
877
|
-
record,
|
|
878
|
-
offset,
|
|
879
|
-
record.length - offset
|
|
880
|
-
);
|
|
881
|
-
if (!Number.isInteger(written) || written <= 0) {
|
|
882
|
-
throw new Error('preview phase outbox write made no progress');
|
|
1881
|
+
if (committedBytes + record.length > maxRunBytes) {
|
|
1882
|
+
const error = new Error(
|
|
1883
|
+
`preview phase run quota would exceed ${maxRunBytes} bytes`
|
|
1884
|
+
);
|
|
1885
|
+
error.code = 'PREVIEW_PHASE_RUN_QUOTA_EXCEEDED';
|
|
1886
|
+
throw error;
|
|
883
1887
|
}
|
|
884
|
-
offset
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
1888
|
+
let offset = 0;
|
|
1889
|
+
while (offset < record.length) {
|
|
1890
|
+
const written = fileSystem.writeSync(
|
|
1891
|
+
eventsFD,
|
|
1892
|
+
record,
|
|
1893
|
+
offset,
|
|
1894
|
+
record.length - offset
|
|
1895
|
+
);
|
|
1896
|
+
if (!Number.isInteger(written) || written <= 0) {
|
|
1897
|
+
throw new Error('preview phase outbox write made no progress');
|
|
1898
|
+
}
|
|
1899
|
+
offset += written;
|
|
1900
|
+
}
|
|
1901
|
+
committedBytes += record.length;
|
|
1902
|
+
return true;
|
|
1903
|
+
} catch (error) {
|
|
1904
|
+
if (eventsFD != null) {
|
|
1905
|
+
try {
|
|
1906
|
+
fileSystem.closeSync(eventsFD);
|
|
1907
|
+
} catch {
|
|
1908
|
+
// Best-effort fd cleanup after append failure.
|
|
1909
|
+
}
|
|
1910
|
+
eventsFD = null;
|
|
1911
|
+
committedBytes = null;
|
|
1912
|
+
recovered = false;
|
|
1913
|
+
}
|
|
1914
|
+
if (error?.code === 'PREVIEW_PHASE_PRODUCER_EPOCH_CONFLICT') {
|
|
1915
|
+
try {
|
|
1916
|
+
finalizeUnboundFallback(event, error);
|
|
1917
|
+
} catch {
|
|
1918
|
+
// The controller can finish a dead CAS loser from its immutable
|
|
1919
|
+
// writer identity. Preserve the original conflict for fallback.
|
|
1920
|
+
}
|
|
1921
|
+
throw error;
|
|
1922
|
+
}
|
|
1923
|
+
if (error?.code === 'PREVIEW_PHASE_RUN_CONFLICT') {
|
|
1924
|
+
try {
|
|
1925
|
+
finalizeUnpublishedStagingFallback(event, error);
|
|
1926
|
+
} catch {
|
|
1927
|
+
// Preserve the staging writer for controller-owned dead-owner GC.
|
|
1928
|
+
}
|
|
1929
|
+
throw error;
|
|
1930
|
+
}
|
|
1931
|
+
if (
|
|
1932
|
+
isTransientFilesystemError(error) &&
|
|
1933
|
+
attempt < MAX_TRANSIENT_ATTEMPTS
|
|
1934
|
+
) {
|
|
1935
|
+
continue;
|
|
1936
|
+
}
|
|
1937
|
+
latchDegraded(event, error);
|
|
1938
|
+
throw error;
|
|
896
1939
|
}
|
|
897
|
-
latchDegraded(event, error);
|
|
898
|
-
throw error;
|
|
899
1940
|
}
|
|
1941
|
+
throw new Error('preview phase transient retry loop exhausted');
|
|
900
1942
|
}
|
|
901
1943
|
|
|
902
1944
|
function flush() {
|
|
903
1945
|
if (eventsFD == null) return true;
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
1946
|
+
for (let attempt = 1; attempt <= MAX_TRANSIENT_ATTEMPTS; attempt += 1) {
|
|
1947
|
+
try {
|
|
1948
|
+
fileSystem.fdatasyncSync(eventsFD);
|
|
1949
|
+
return true;
|
|
1950
|
+
} catch (error) {
|
|
1951
|
+
if (
|
|
1952
|
+
isTransientFilesystemError(error) &&
|
|
1953
|
+
attempt < MAX_TRANSIENT_ATTEMPTS
|
|
1954
|
+
) {
|
|
1955
|
+
continue;
|
|
1956
|
+
}
|
|
1957
|
+
latchDegraded(null, error);
|
|
1958
|
+
throw error;
|
|
1959
|
+
}
|
|
910
1960
|
}
|
|
1961
|
+
throw new Error('preview phase flush retry loop exhausted');
|
|
911
1962
|
}
|
|
912
1963
|
|
|
913
1964
|
function pollQuiesce() {
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
if (
|
|
919
|
-
|
|
1965
|
+
const state = (localSealed, canonicalAcknowledged) => ({
|
|
1966
|
+
local_sealed: localSealed,
|
|
1967
|
+
canonical_acknowledged: canonicalAcknowledged,
|
|
1968
|
+
});
|
|
1969
|
+
if (closed) {
|
|
1970
|
+
return state(closeResult, everOwnedCanonicalWriter && closeResult);
|
|
1971
|
+
}
|
|
1972
|
+
if (sealRequested) {
|
|
1973
|
+
const localSealed = close();
|
|
1974
|
+
return state(localSealed, everOwnedCanonicalWriter && localSealed);
|
|
1975
|
+
}
|
|
1976
|
+
if (!metadata) return state(false, false);
|
|
1977
|
+
if (!fileSystem.existsSync(quiesceRequestPath)) return state(false, false);
|
|
920
1978
|
const request = readRegularJSON(quiesceRequestPath, 'quiesce request');
|
|
921
1979
|
assertStateGeneration(request, 'quiesce request');
|
|
922
1980
|
if (
|
|
@@ -930,11 +1988,29 @@ function createPreviewPhaseOutbox(options) {
|
|
|
930
1988
|
`invalid preview phase quiesce request: ${quiesceRequestPath}`
|
|
931
1989
|
);
|
|
932
1990
|
}
|
|
933
|
-
|
|
1991
|
+
if (!ownsSelectedWriterLock()) {
|
|
1992
|
+
// Do not acknowledge or mutate the canonical owner lifecycle. The
|
|
1993
|
+
// quiesce boundary still terminalizes this producer's own fallback file.
|
|
1994
|
+
sealRequested = true;
|
|
1995
|
+
closeResult = sealFallback();
|
|
1996
|
+
closed = true;
|
|
1997
|
+
return state(closeResult, false);
|
|
1998
|
+
}
|
|
1999
|
+
const localSealed = close();
|
|
2000
|
+
return state(localSealed, localSealed);
|
|
934
2001
|
}
|
|
935
2002
|
|
|
936
2003
|
function close() {
|
|
937
|
-
|
|
2004
|
+
const fallbackCloseResult = sealFallback();
|
|
2005
|
+
// A fallback-only producer never owns the canonical writer lifecycle.
|
|
2006
|
+
// Its open spool fd is sufficient to terminalize after controller rename;
|
|
2007
|
+
// attempting canonical close here could recreate or mutate another owner.
|
|
2008
|
+
if (fallbackStarted && !everOwnedCanonicalWriter) {
|
|
2009
|
+
closed = true;
|
|
2010
|
+
closeResult = fallbackCloseResult;
|
|
2011
|
+
return closeResult;
|
|
2012
|
+
}
|
|
2013
|
+
if (closed) return closeResult && fallbackCloseResult;
|
|
938
2014
|
sealRequested = true;
|
|
939
2015
|
let finalSize = null;
|
|
940
2016
|
try {
|
|
@@ -943,18 +2019,17 @@ function createPreviewPhaseOutbox(options) {
|
|
|
943
2019
|
// this initialization when the filesystem becomes available again.
|
|
944
2020
|
if (!metadata && !degraded && !fallbackRequired) {
|
|
945
2021
|
closed = true;
|
|
946
|
-
closeResult =
|
|
947
|
-
return
|
|
2022
|
+
closeResult = fallbackCloseResult;
|
|
2023
|
+
return closeResult;
|
|
948
2024
|
}
|
|
949
2025
|
assertSafeDirectory(fileSystem, root);
|
|
950
2026
|
assertRootDirectoryQuota();
|
|
951
|
-
|
|
952
|
-
ensureMetadata();
|
|
2027
|
+
ensureCanonicalWriter();
|
|
953
2028
|
|
|
954
2029
|
if (fileSystem.existsSync(sealPath)) {
|
|
955
2030
|
const seal = readAndValidateSeal();
|
|
956
2031
|
fallbackRequired ||= seal.fallback_required === true;
|
|
957
|
-
if (
|
|
2032
|
+
if (ownsSelectedWriterLock()) {
|
|
958
2033
|
publishActiveProducerMarker();
|
|
959
2034
|
publishTerminalProducerMarker();
|
|
960
2035
|
removeOwnedWriterLock();
|
|
@@ -976,14 +2051,15 @@ function createPreviewPhaseOutbox(options) {
|
|
|
976
2051
|
producerMarkerTerminal = true;
|
|
977
2052
|
}
|
|
978
2053
|
closed = true;
|
|
979
|
-
closeResult =
|
|
980
|
-
return
|
|
2054
|
+
closeResult = fallbackCloseResult;
|
|
2055
|
+
return closeResult;
|
|
981
2056
|
}
|
|
982
2057
|
|
|
983
|
-
if (!
|
|
2058
|
+
if (!ownsSelectedWriterLock()) {
|
|
984
2059
|
acquireWriterLock();
|
|
985
2060
|
}
|
|
986
2061
|
publishActiveProducerMarker();
|
|
2062
|
+
observeFallbackIntent();
|
|
987
2063
|
observeDegradedMarker();
|
|
988
2064
|
if (eventsFD != null) {
|
|
989
2065
|
fileSystem.fdatasyncSync(eventsFD);
|
|
@@ -1021,36 +2097,44 @@ function createPreviewPhaseOutbox(options) {
|
|
|
1021
2097
|
});
|
|
1022
2098
|
publishTerminalProducerMarker();
|
|
1023
2099
|
removeOwnedWriterLock();
|
|
1024
|
-
closeResult =
|
|
2100
|
+
closeResult = fallbackCloseResult;
|
|
1025
2101
|
closed = true;
|
|
1026
|
-
} catch {
|
|
2102
|
+
} catch (error) {
|
|
1027
2103
|
closeResult = false;
|
|
1028
|
-
|
|
2104
|
+
if (!isTransientFilesystemError(error)) {
|
|
2105
|
+
latchDegraded(null, error);
|
|
2106
|
+
}
|
|
1029
2107
|
if (eventsFD != null) {
|
|
1030
2108
|
try {
|
|
1031
2109
|
fileSystem.closeSync(eventsFD);
|
|
1032
|
-
} catch {
|
|
2110
|
+
} catch {
|
|
2111
|
+
// Best-effort fd cleanup after close failure.
|
|
2112
|
+
}
|
|
1033
2113
|
eventsFD = null;
|
|
1034
2114
|
committedBytes = null;
|
|
1035
2115
|
}
|
|
1036
|
-
// Keep the unique writer owner and pending close intent for
|
|
1037
|
-
//
|
|
1038
|
-
//
|
|
2116
|
+
// Keep the unique writer owner and pending close intent for retries. If
|
|
2117
|
+
// the process dies, shell-server is the sole authority that verifies
|
|
2118
|
+
// PID+starttime, repairs the tail, and seals the generation.
|
|
1039
2119
|
}
|
|
1040
2120
|
return closeResult;
|
|
1041
2121
|
}
|
|
1042
2122
|
|
|
1043
2123
|
return {
|
|
1044
2124
|
append,
|
|
2125
|
+
append_fallback: appendFallback,
|
|
2126
|
+
seal_fallback: sealFallback,
|
|
1045
2127
|
flush,
|
|
1046
2128
|
close,
|
|
1047
2129
|
poll_quiesce: pollQuiesce,
|
|
2130
|
+
is_local_sealed: () => closed && closeResult,
|
|
1048
2131
|
events_path: eventsPath,
|
|
1049
2132
|
meta_path: metaPath,
|
|
1050
2133
|
seal_path: sealPath,
|
|
1051
2134
|
degraded_path: degradedPath,
|
|
1052
2135
|
quiesce_request_path: quiesceRequestPath,
|
|
1053
2136
|
producer_epoch_marker_path: producerEpochMarkerPath,
|
|
2137
|
+
fallback_spool_path: fallbackSpoolPath,
|
|
1054
2138
|
};
|
|
1055
2139
|
}
|
|
1056
2140
|
|
|
@@ -1061,6 +2145,11 @@ module.exports = {
|
|
|
1061
2145
|
DEFAULT_MAX_ROOT_DIRECTORIES,
|
|
1062
2146
|
DEFAULT_MAX_ROOT_SCAN_DIRECTORIES,
|
|
1063
2147
|
DEFAULT_MAX_RUN_SCAN_ENTRIES,
|
|
2148
|
+
DEFAULT_MAX_FALLBACK_FILES,
|
|
2149
|
+
DEFAULT_MAX_FALLBACK_BYTES,
|
|
2150
|
+
DEFAULT_MAX_FALLBACK_EVENTS,
|
|
2151
|
+
MAX_FALLBACK_RECORD_BYTES,
|
|
1064
2152
|
MAX_EVENT_BYTES,
|
|
2153
|
+
createFallbackSessionID,
|
|
1065
2154
|
createPreviewPhaseOutbox,
|
|
1066
2155
|
};
|