@forgeax/engine-ddc 0.0.0-dev.8d955ade1c79
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +147 -0
- package/dist/.tsbuildinfo +1 -0
- package/dist/__tests__/concurrency.integration.test.d.ts +2 -0
- package/dist/__tests__/concurrency.integration.test.d.ts.map +1 -0
- package/dist/__tests__/consumer-path.integration.test.d.ts +2 -0
- package/dist/__tests__/consumer-path.integration.test.d.ts.map +1 -0
- package/dist/__tests__/crash-recovery.integration.test.d.ts +2 -0
- package/dist/__tests__/crash-recovery.integration.test.d.ts.map +1 -0
- package/dist/__tests__/entry-store.integration.test.d.ts +2 -0
- package/dist/__tests__/entry-store.integration.test.d.ts.map +1 -0
- package/dist/__tests__/errors.unit.test.d.ts +2 -0
- package/dist/__tests__/errors.unit.test.d.ts.map +1 -0
- package/dist/__tests__/generation-session.integration.test.d.ts +2 -0
- package/dist/__tests__/generation-session.integration.test.d.ts.map +1 -0
- package/dist/__tests__/generation-session.unit.test.d.ts +2 -0
- package/dist/__tests__/generation-session.unit.test.d.ts.map +1 -0
- package/dist/__tests__/key.unit.test.d.ts +2 -0
- package/dist/__tests__/key.unit.test.d.ts.map +1 -0
- package/dist/__tests__/layout.unit.test.d.ts +2 -0
- package/dist/__tests__/layout.unit.test.d.ts.map +1 -0
- package/dist/__tests__/lifecycle.unit.test.d.ts +2 -0
- package/dist/__tests__/lifecycle.unit.test.d.ts.map +1 -0
- package/dist/__tests__/multiprocess-gc.integration.test.d.ts +2 -0
- package/dist/__tests__/multiprocess-gc.integration.test.d.ts.map +1 -0
- package/dist/__tests__/multiprocess-lifecycle.integration.test.d.ts +2 -0
- package/dist/__tests__/multiprocess-lifecycle.integration.test.d.ts.map +1 -0
- package/dist/__tests__/multiprocess-worker.d.ts +2 -0
- package/dist/__tests__/multiprocess-worker.d.ts.map +1 -0
- package/dist/__tests__/owner-chain.integration.test.d.ts +2 -0
- package/dist/__tests__/owner-chain.integration.test.d.ts.map +1 -0
- package/dist/__tests__/status-root-kind-owner.test-d.d.ts +2 -0
- package/dist/__tests__/status-root-kind-owner.test-d.d.ts.map +1 -0
- package/dist/__tests__/status.unit.test.d.ts +2 -0
- package/dist/__tests__/status.unit.test.d.ts.map +1 -0
- package/dist/build-cache.d.ts +18 -0
- package/dist/build-cache.d.ts.map +1 -0
- package/dist/entry-store.d.ts +53 -0
- package/dist/entry-store.d.ts.map +1 -0
- package/dist/entry-store.mjs +349 -0
- package/dist/entry-store.mjs.map +1 -0
- package/dist/errors.d.ts +58 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.mjs +109 -0
- package/dist/errors.mjs.map +1 -0
- package/dist/gc.d.ts +13 -0
- package/dist/gc.d.ts.map +1 -0
- package/dist/index.d.ts +12 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.mjs +1574 -0
- package/dist/index.mjs.map +1 -0
- package/dist/key.d.ts +13 -0
- package/dist/key.d.ts.map +1 -0
- package/dist/key.mjs +38 -0
- package/dist/key.mjs.map +1 -0
- package/dist/layout.d.ts +51 -0
- package/dist/layout.d.ts.map +1 -0
- package/dist/layout.mjs +68 -0
- package/dist/layout.mjs.map +1 -0
- package/dist/lifecycle.d.ts +93 -0
- package/dist/lifecycle.d.ts.map +1 -0
- package/dist/publication.d.ts +50 -0
- package/dist/publication.d.ts.map +1 -0
- package/dist/runtime-scope.d.ts +10 -0
- package/dist/runtime-scope.d.ts.map +1 -0
- package/dist/session.d.ts +46 -0
- package/dist/session.d.ts.map +1 -0
- package/dist/status.d.ts +39 -0
- package/dist/status.d.ts.map +1 -0
- package/dist/status.mjs +20 -0
- package/dist/status.mjs.map +1 -0
- package/package.json +101 -0
- package/src/__tests__/concurrency.integration.test.ts +53 -0
- package/src/__tests__/consumer-path.integration.test.ts +61 -0
- package/src/__tests__/crash-recovery.integration.test.ts +84 -0
- package/src/__tests__/entry-store.integration.test.ts +83 -0
- package/src/__tests__/errors.unit.test.ts +54 -0
- package/src/__tests__/generation-session.integration.test.ts +202 -0
- package/src/__tests__/generation-session.unit.test.ts +35 -0
- package/src/__tests__/key.unit.test.ts +49 -0
- package/src/__tests__/layout.unit.test.ts +50 -0
- package/src/__tests__/lifecycle.unit.test.ts +258 -0
- package/src/__tests__/multiprocess-gc.integration.test.ts +36 -0
- package/src/__tests__/multiprocess-lifecycle.integration.test.ts +129 -0
- package/src/__tests__/multiprocess-worker.ts +41 -0
- package/src/__tests__/owner-chain.integration.test.ts +45 -0
- package/src/__tests__/status-root-kind-owner.test-d.ts +42 -0
- package/src/__tests__/status.unit.test.ts +56 -0
- package/src/build-cache.ts +60 -0
- package/src/entry-store.ts +317 -0
- package/src/errors.ts +164 -0
- package/src/gc.ts +34 -0
- package/src/index.ts +92 -0
- package/src/key.ts +46 -0
- package/src/layout.ts +132 -0
- package/src/lifecycle.ts +792 -0
- package/src/publication.ts +446 -0
- package/src/runtime-scope.ts +87 -0
- package/src/session.ts +281 -0
- package/src/status.ts +58 -0
package/src/lifecycle.ts
ADDED
|
@@ -0,0 +1,792 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { DdcEntryStore } from './entry-store.js';
|
|
5
|
+
import { DdcStoreError } from './errors.js';
|
|
6
|
+
|
|
7
|
+
export type DdcLifecycleState = 'missing' | 'cooking' | 'current' | 'stale' | 'failed';
|
|
8
|
+
|
|
9
|
+
export interface DdcLease {
|
|
10
|
+
readonly guid: string;
|
|
11
|
+
readonly desiredKey: string;
|
|
12
|
+
readonly attempt: string;
|
|
13
|
+
readonly generation: number;
|
|
14
|
+
readonly expectedRevision: number;
|
|
15
|
+
readonly instanceId: string;
|
|
16
|
+
readonly expiresAt: number;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface DdcHead {
|
|
20
|
+
readonly guid: string;
|
|
21
|
+
readonly desiredKey: string;
|
|
22
|
+
readonly state: DdcLifecycleState;
|
|
23
|
+
readonly currentKey: string | undefined;
|
|
24
|
+
readonly lastKnownGoodKey: string | undefined;
|
|
25
|
+
readonly revision?: number;
|
|
26
|
+
readonly generation?: number;
|
|
27
|
+
readonly activeLease?: DdcLease;
|
|
28
|
+
readonly failure?: { readonly code: string; readonly detail: string };
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export interface DdcCommitResult {
|
|
32
|
+
readonly result: 'current' | 'stale' | 'lease-lost' | 'invalid';
|
|
33
|
+
readonly key: string;
|
|
34
|
+
readonly revision?: number;
|
|
35
|
+
/** Internal CAS evidence retained for a later generation rollback. */
|
|
36
|
+
readonly restoreFence?: DdcRestoreFence;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface DdcRestoreFence {
|
|
40
|
+
readonly attempt: string;
|
|
41
|
+
readonly generation: number;
|
|
42
|
+
readonly revision: number;
|
|
43
|
+
readonly desiredKey: string;
|
|
44
|
+
readonly outcome: 'current' | 'invalid' | 'failed';
|
|
45
|
+
readonly key?: string;
|
|
46
|
+
readonly currentKey?: string;
|
|
47
|
+
readonly lastKnownGoodKey?: string;
|
|
48
|
+
readonly failure?: { readonly code: string; readonly detail: string };
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export interface DdcRestoreResult {
|
|
52
|
+
readonly result: 'restored' | 'not-owner';
|
|
53
|
+
readonly revision: number;
|
|
54
|
+
readonly generation?: number;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export type DdcRollbackSnapshot = Omit<DdcHead, 'activeLease'>;
|
|
58
|
+
|
|
59
|
+
export interface DdcBeginResult {
|
|
60
|
+
readonly lease: DdcLease;
|
|
61
|
+
readonly previousHead: DdcRollbackSnapshot;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
interface HeadRecord {
|
|
65
|
+
readonly guid: string;
|
|
66
|
+
readonly desiredKey: string;
|
|
67
|
+
readonly revision: number;
|
|
68
|
+
readonly currentKey?: string;
|
|
69
|
+
readonly lastKnownGoodKey?: string;
|
|
70
|
+
readonly generation?: number;
|
|
71
|
+
readonly active?: DdcLease;
|
|
72
|
+
readonly supersededAttempts?: readonly string[];
|
|
73
|
+
readonly stale?: boolean;
|
|
74
|
+
readonly empty?: boolean;
|
|
75
|
+
readonly failure?: {
|
|
76
|
+
readonly desiredKey: string;
|
|
77
|
+
readonly code: string;
|
|
78
|
+
readonly detail: string;
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
interface GenerationRecord {
|
|
83
|
+
readonly schemaVersion: 'forgeax-ddc-generation/v2';
|
|
84
|
+
readonly next: number;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const LOCK_WAIT_MS = 10;
|
|
88
|
+
const LOCK_TIMEOUT_MS = 5000;
|
|
89
|
+
const LEASE_TTL_MS = 30_000;
|
|
90
|
+
|
|
91
|
+
function headFile(heads: string, guid: string): string {
|
|
92
|
+
return join(heads, `${encodeURIComponent(guid)}.json`);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function lockFile(root: string, name: string): string {
|
|
96
|
+
return join(root, 'locks', `${encodeURIComponent(name)}.lock`);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function withRevision(
|
|
100
|
+
result: Omit<DdcCommitResult, 'revision'>,
|
|
101
|
+
revision: number,
|
|
102
|
+
restoreFence?: DdcRestoreFence,
|
|
103
|
+
): DdcCommitResult {
|
|
104
|
+
const value = { ...result } as DdcCommitResult;
|
|
105
|
+
Object.defineProperty(value, 'revision', { value: revision, enumerable: false });
|
|
106
|
+
if (restoreFence !== undefined) {
|
|
107
|
+
Object.defineProperty(value, 'restoreFence', { value: restoreFence, enumerable: false });
|
|
108
|
+
}
|
|
109
|
+
return value;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
async function delay(ms: number): Promise<void> {
|
|
113
|
+
await new Promise((resolveDelay) => setTimeout(resolveDelay, ms));
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
117
|
+
return value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function isString(value: unknown): value is string {
|
|
121
|
+
return typeof value === 'string';
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function isNonNegativeInteger(value: unknown): value is number {
|
|
125
|
+
return typeof value === 'number' && Number.isInteger(value) && value >= 0;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function isLeaseRecord(value: unknown): value is DdcLease {
|
|
129
|
+
if (!isRecord(value)) return false;
|
|
130
|
+
return (
|
|
131
|
+
isString(value.guid) &&
|
|
132
|
+
isString(value.desiredKey) &&
|
|
133
|
+
isString(value.attempt) &&
|
|
134
|
+
isNonNegativeInteger(value.generation) &&
|
|
135
|
+
isNonNegativeInteger(value.expectedRevision) &&
|
|
136
|
+
isString(value.instanceId) &&
|
|
137
|
+
typeof value.expiresAt === 'number' &&
|
|
138
|
+
Number.isFinite(value.expiresAt)
|
|
139
|
+
);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function isHeadRecord(value: unknown): value is HeadRecord {
|
|
143
|
+
if (!isRecord(value)) return false;
|
|
144
|
+
if (
|
|
145
|
+
!isString(value.guid) ||
|
|
146
|
+
!isString(value.desiredKey) ||
|
|
147
|
+
!isNonNegativeInteger(value.revision)
|
|
148
|
+
) {
|
|
149
|
+
return false;
|
|
150
|
+
}
|
|
151
|
+
if (value.currentKey !== undefined && !isString(value.currentKey)) return false;
|
|
152
|
+
if (value.lastKnownGoodKey !== undefined && !isString(value.lastKnownGoodKey)) return false;
|
|
153
|
+
if (
|
|
154
|
+
value.generation !== undefined &&
|
|
155
|
+
(!isNonNegativeInteger(value.generation) || value.generation === 0)
|
|
156
|
+
) {
|
|
157
|
+
return false;
|
|
158
|
+
}
|
|
159
|
+
if (value.active !== undefined && !isLeaseRecord(value.active)) return false;
|
|
160
|
+
if (
|
|
161
|
+
value.supersededAttempts !== undefined &&
|
|
162
|
+
(!Array.isArray(value.supersededAttempts) || !value.supersededAttempts.every(isString))
|
|
163
|
+
) {
|
|
164
|
+
return false;
|
|
165
|
+
}
|
|
166
|
+
if (value.stale !== undefined && typeof value.stale !== 'boolean') return false;
|
|
167
|
+
if (value.empty !== undefined && typeof value.empty !== 'boolean') return false;
|
|
168
|
+
if (value.failure !== undefined) {
|
|
169
|
+
if (!isRecord(value.failure)) return false;
|
|
170
|
+
if (
|
|
171
|
+
!isString(value.failure.desiredKey) ||
|
|
172
|
+
!isString(value.failure.code) ||
|
|
173
|
+
!isString(value.failure.detail)
|
|
174
|
+
) {
|
|
175
|
+
return false;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return true;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function malformedHeadError(
|
|
182
|
+
actual: 'syntax-invalid' | 'schema-invalid' | 'unreadable',
|
|
183
|
+
): DdcStoreError {
|
|
184
|
+
const detail =
|
|
185
|
+
actual === 'syntax-invalid'
|
|
186
|
+
? 'DDC head JSON is syntactically invalid'
|
|
187
|
+
: actual === 'schema-invalid'
|
|
188
|
+
? 'DDC head record does not match the lifecycle schema'
|
|
189
|
+
: 'DDC head file is not readable';
|
|
190
|
+
return new DdcStoreError({
|
|
191
|
+
code: 'ddc-head-conflict',
|
|
192
|
+
detail,
|
|
193
|
+
expected: 'a valid DDC head record',
|
|
194
|
+
actual,
|
|
195
|
+
hint: 'inspect the current head and retry with a fresh revision',
|
|
196
|
+
owner: 'engine-ddc',
|
|
197
|
+
rootKind: 'project-ddc',
|
|
198
|
+
recoveryActions: [
|
|
199
|
+
{ kind: 'inspect', executable: true },
|
|
200
|
+
{ kind: 'retry', executable: true },
|
|
201
|
+
],
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/** Cross-process mkdir lock; a lock owned by a dead PID is reclaimable. */
|
|
206
|
+
export async function withDdcLock<T>(
|
|
207
|
+
root: string,
|
|
208
|
+
name: string,
|
|
209
|
+
operation: () => Promise<T>,
|
|
210
|
+
): Promise<T> {
|
|
211
|
+
const path = lockFile(root, name);
|
|
212
|
+
await mkdir(join(root, 'locks'), { recursive: true });
|
|
213
|
+
const deadline = Date.now() + LOCK_TIMEOUT_MS;
|
|
214
|
+
while (true) {
|
|
215
|
+
try {
|
|
216
|
+
await mkdir(path);
|
|
217
|
+
await writeFile(
|
|
218
|
+
join(path, 'owner.json'),
|
|
219
|
+
JSON.stringify({ pid: process.pid, acquiredAt: Date.now() }),
|
|
220
|
+
);
|
|
221
|
+
try {
|
|
222
|
+
return await operation();
|
|
223
|
+
} finally {
|
|
224
|
+
await rm(path, { recursive: true, force: true });
|
|
225
|
+
}
|
|
226
|
+
} catch (error) {
|
|
227
|
+
if ((error as { code?: string }).code !== 'EEXIST') throw error;
|
|
228
|
+
let ownerPid: number | undefined;
|
|
229
|
+
try {
|
|
230
|
+
ownerPid = (
|
|
231
|
+
JSON.parse(await readFile(join(path, 'owner.json'), 'utf8')) as { pid?: number }
|
|
232
|
+
).pid;
|
|
233
|
+
} catch {
|
|
234
|
+
ownerPid = undefined;
|
|
235
|
+
}
|
|
236
|
+
// mkdir() publishes the lock directory before the owner record can be
|
|
237
|
+
// written. Treat that short publication window as an in-flight lock,
|
|
238
|
+
// not as a dead owner: reclaiming it here lets two processes enter the
|
|
239
|
+
// same critical section and loses the superseded-attempt fence.
|
|
240
|
+
if (ownerPid === undefined) {
|
|
241
|
+
if (Date.now() >= deadline) {
|
|
242
|
+
throw new DdcStoreError({
|
|
243
|
+
code: 'ddc-lease-expired',
|
|
244
|
+
detail: `DDC lock ${name} did not publish an owner record`,
|
|
245
|
+
hint: 'inspect the lock owner and retry after the incomplete instance exits',
|
|
246
|
+
expected: 'a lock owner record',
|
|
247
|
+
actual: { path },
|
|
248
|
+
rootKind: 'project-ddc',
|
|
249
|
+
});
|
|
250
|
+
}
|
|
251
|
+
await delay(LOCK_WAIT_MS);
|
|
252
|
+
continue;
|
|
253
|
+
}
|
|
254
|
+
let alive = false;
|
|
255
|
+
try {
|
|
256
|
+
process.kill(ownerPid, 0);
|
|
257
|
+
alive = true;
|
|
258
|
+
} catch {
|
|
259
|
+
alive = false;
|
|
260
|
+
}
|
|
261
|
+
if (!alive) {
|
|
262
|
+
await rm(path, { recursive: true, force: true });
|
|
263
|
+
continue;
|
|
264
|
+
}
|
|
265
|
+
if (Date.now() >= deadline) {
|
|
266
|
+
throw new DdcStoreError({
|
|
267
|
+
code: 'ddc-lease-expired',
|
|
268
|
+
detail: `DDC lock ${name} remained owned by process ${ownerPid}`,
|
|
269
|
+
hint: 'inspect the owner and retry after the stale instance exits',
|
|
270
|
+
expected: 'an available project lock',
|
|
271
|
+
actual: { ownerPid },
|
|
272
|
+
rootKind: 'project-ddc',
|
|
273
|
+
lease: String(ownerPid),
|
|
274
|
+
});
|
|
275
|
+
}
|
|
276
|
+
await delay(LOCK_WAIT_MS);
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
export class DdcLifecycle {
|
|
282
|
+
private readonly heads: string;
|
|
283
|
+
private readonly entries: DdcEntryStore;
|
|
284
|
+
private readonly root: string;
|
|
285
|
+
private readonly leaseTtlMs: number;
|
|
286
|
+
|
|
287
|
+
public constructor(root: string, options?: { readonly leaseTtlMs?: number }) {
|
|
288
|
+
this.root = root;
|
|
289
|
+
this.heads = join(root, 'heads');
|
|
290
|
+
this.entries = new DdcEntryStore(root);
|
|
291
|
+
this.leaseTtlMs = options?.leaseTtlMs ?? LEASE_TTL_MS;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
public async inspect(guid: string, desiredKey: string): Promise<DdcHead> {
|
|
295
|
+
const record = await this.read(guid);
|
|
296
|
+
return this.projectHead(guid, desiredKey, record);
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/** Begin a lease and capture the accepted rollback snapshot under one head lock. */
|
|
300
|
+
public async beginWithSnapshot(guid: string, desiredKey: string): Promise<DdcBeginResult> {
|
|
301
|
+
return withDdcLock(this.root, `head-${guid}`, async () => {
|
|
302
|
+
const previous = await this.read(guid);
|
|
303
|
+
const previousHead = await this.projectHead(guid, desiredKey, previous, false);
|
|
304
|
+
const generation = await this.allocateGeneration(previous?.generation ?? 0);
|
|
305
|
+
const lease: DdcLease = {
|
|
306
|
+
guid,
|
|
307
|
+
desiredKey,
|
|
308
|
+
attempt: randomUUID(),
|
|
309
|
+
generation,
|
|
310
|
+
expectedRevision: previous?.revision ?? 0,
|
|
311
|
+
instanceId: `${process.pid}:${randomUUID()}`,
|
|
312
|
+
expiresAt: Date.now() + this.leaseTtlMs,
|
|
313
|
+
};
|
|
314
|
+
const lastKnownGoodKey =
|
|
315
|
+
previous?.currentKey !== undefined && previous.currentKey !== desiredKey
|
|
316
|
+
? previous.currentKey
|
|
317
|
+
: previous?.lastKnownGoodKey;
|
|
318
|
+
const supersededAttempts = [
|
|
319
|
+
...(previous?.supersededAttempts ?? []),
|
|
320
|
+
...(previous?.active === undefined ? [] : [previous.active.attempt]),
|
|
321
|
+
];
|
|
322
|
+
await this.write({
|
|
323
|
+
guid,
|
|
324
|
+
desiredKey,
|
|
325
|
+
revision: previous?.revision ?? 0,
|
|
326
|
+
generation,
|
|
327
|
+
active: lease,
|
|
328
|
+
...(previous?.currentKey === undefined ? {} : { currentKey: previous.currentKey }),
|
|
329
|
+
...(supersededAttempts.length === 0 ? {} : { supersededAttempts }),
|
|
330
|
+
...(lastKnownGoodKey === undefined ? {} : { lastKnownGoodKey }),
|
|
331
|
+
});
|
|
332
|
+
return { lease, previousHead };
|
|
333
|
+
});
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
public async begin(guid: string, desiredKey: string): Promise<DdcLease> {
|
|
337
|
+
return (await this.beginWithSnapshot(guid, desiredKey)).lease;
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
public async commit(lease: DdcLease, validatedKey: string): Promise<DdcCommitResult> {
|
|
341
|
+
return withDdcLock(this.root, `head-${lease.guid}`, async () => {
|
|
342
|
+
const current = await this.read(lease.guid);
|
|
343
|
+
if (current?.active?.attempt !== lease.attempt) {
|
|
344
|
+
if (current?.supersededAttempts?.includes(lease.attempt)) {
|
|
345
|
+
await this.write({ ...current, stale: true });
|
|
346
|
+
return withRevision({ result: 'stale', key: validatedKey }, current.revision);
|
|
347
|
+
}
|
|
348
|
+
return withRevision({ result: 'lease-lost', key: validatedKey }, current?.revision ?? 0);
|
|
349
|
+
}
|
|
350
|
+
if (Date.now() > lease.expiresAt) {
|
|
351
|
+
throw new DdcStoreError({
|
|
352
|
+
code: 'ddc-lease-expired',
|
|
353
|
+
detail: 'cook lease expired before commit fencing',
|
|
354
|
+
expected: { attempt: lease.attempt, revision: lease.expectedRevision },
|
|
355
|
+
actual: { revision: current.revision },
|
|
356
|
+
owner: 'engine-ddc',
|
|
357
|
+
rootKind: 'project-ddc',
|
|
358
|
+
generation: lease.generation,
|
|
359
|
+
lease: lease.attempt,
|
|
360
|
+
revision: current.revision,
|
|
361
|
+
});
|
|
362
|
+
}
|
|
363
|
+
if (
|
|
364
|
+
current.revision !== lease.expectedRevision ||
|
|
365
|
+
current.desiredKey !== lease.desiredKey ||
|
|
366
|
+
validatedKey !== lease.desiredKey
|
|
367
|
+
) {
|
|
368
|
+
await this.write({ ...current, stale: true });
|
|
369
|
+
return withRevision({ result: 'stale', key: validatedKey }, current.revision);
|
|
370
|
+
}
|
|
371
|
+
const entry = await this.entries.read(validatedKey);
|
|
372
|
+
if (entry === null || entry.guid !== lease.guid || entry.receipt.key !== validatedKey) {
|
|
373
|
+
const nextRevision = current.revision + 1;
|
|
374
|
+
const failure = {
|
|
375
|
+
desiredKey: lease.desiredKey,
|
|
376
|
+
code: 'entry-invalid',
|
|
377
|
+
detail: 'validated DDC key has no readable entry for this asset',
|
|
378
|
+
};
|
|
379
|
+
await this.write({
|
|
380
|
+
guid: lease.guid,
|
|
381
|
+
desiredKey: lease.desiredKey,
|
|
382
|
+
revision: nextRevision,
|
|
383
|
+
generation: lease.generation,
|
|
384
|
+
...(current.lastKnownGoodKey === undefined
|
|
385
|
+
? {}
|
|
386
|
+
: { lastKnownGoodKey: current.lastKnownGoodKey }),
|
|
387
|
+
...(current.supersededAttempts === undefined
|
|
388
|
+
? {}
|
|
389
|
+
: { supersededAttempts: current.supersededAttempts }),
|
|
390
|
+
failure,
|
|
391
|
+
});
|
|
392
|
+
return withRevision({ result: 'invalid', key: validatedKey }, nextRevision, {
|
|
393
|
+
attempt: lease.attempt,
|
|
394
|
+
generation: lease.generation,
|
|
395
|
+
revision: nextRevision,
|
|
396
|
+
desiredKey: lease.desiredKey,
|
|
397
|
+
outcome: 'invalid',
|
|
398
|
+
...(current.lastKnownGoodKey === undefined
|
|
399
|
+
? {}
|
|
400
|
+
: { lastKnownGoodKey: current.lastKnownGoodKey }),
|
|
401
|
+
failure: { code: failure.code, detail: failure.detail },
|
|
402
|
+
});
|
|
403
|
+
}
|
|
404
|
+
const lastKnownGoodKey =
|
|
405
|
+
current.currentKey !== undefined && current.currentKey !== validatedKey
|
|
406
|
+
? current.currentKey
|
|
407
|
+
: current.lastKnownGoodKey;
|
|
408
|
+
const nextRevision = current.revision + 1;
|
|
409
|
+
await this.write({
|
|
410
|
+
guid: lease.guid,
|
|
411
|
+
desiredKey: lease.desiredKey,
|
|
412
|
+
revision: nextRevision,
|
|
413
|
+
generation: lease.generation,
|
|
414
|
+
currentKey: validatedKey,
|
|
415
|
+
stale: false,
|
|
416
|
+
...(current.supersededAttempts === undefined
|
|
417
|
+
? {}
|
|
418
|
+
: { supersededAttempts: current.supersededAttempts }),
|
|
419
|
+
...(lastKnownGoodKey === undefined ? {} : { lastKnownGoodKey }),
|
|
420
|
+
});
|
|
421
|
+
return withRevision({ result: 'current', key: validatedKey }, nextRevision, {
|
|
422
|
+
attempt: lease.attempt,
|
|
423
|
+
generation: lease.generation,
|
|
424
|
+
revision: nextRevision,
|
|
425
|
+
desiredKey: lease.desiredKey,
|
|
426
|
+
outcome: 'current',
|
|
427
|
+
key: validatedKey,
|
|
428
|
+
currentKey: validatedKey,
|
|
429
|
+
...(lastKnownGoodKey === undefined ? {} : { lastKnownGoodKey }),
|
|
430
|
+
});
|
|
431
|
+
});
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
public async fail(
|
|
435
|
+
lease: DdcLease,
|
|
436
|
+
failure: { readonly code: string; readonly detail: string },
|
|
437
|
+
): Promise<DdcRestoreFence | undefined> {
|
|
438
|
+
return withDdcLock(this.root, `head-${lease.guid}`, async () => {
|
|
439
|
+
const current = await this.read(lease.guid);
|
|
440
|
+
if (current?.active?.attempt !== lease.attempt) return;
|
|
441
|
+
const nextRevision = current.revision + 1;
|
|
442
|
+
await this.write({
|
|
443
|
+
guid: lease.guid,
|
|
444
|
+
desiredKey: lease.desiredKey,
|
|
445
|
+
revision: nextRevision,
|
|
446
|
+
generation: lease.generation,
|
|
447
|
+
...(current.currentKey === undefined ? {} : { currentKey: current.currentKey }),
|
|
448
|
+
...(current.lastKnownGoodKey === undefined
|
|
449
|
+
? {}
|
|
450
|
+
: { lastKnownGoodKey: current.lastKnownGoodKey }),
|
|
451
|
+
...(current.supersededAttempts === undefined
|
|
452
|
+
? {}
|
|
453
|
+
: { supersededAttempts: current.supersededAttempts }),
|
|
454
|
+
failure: { desiredKey: lease.desiredKey, ...failure },
|
|
455
|
+
});
|
|
456
|
+
return {
|
|
457
|
+
attempt: lease.attempt,
|
|
458
|
+
generation: lease.generation,
|
|
459
|
+
revision: nextRevision,
|
|
460
|
+
desiredKey: lease.desiredKey,
|
|
461
|
+
outcome: 'failed' as const,
|
|
462
|
+
...(current.currentKey === undefined ? {} : { currentKey: current.currentKey }),
|
|
463
|
+
...(current.lastKnownGoodKey === undefined
|
|
464
|
+
? {}
|
|
465
|
+
: { lastKnownGoodKey: current.lastKnownGoodKey }),
|
|
466
|
+
failure,
|
|
467
|
+
};
|
|
468
|
+
});
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
public async heartbeat(lease: DdcLease): Promise<DdcLease> {
|
|
472
|
+
return withDdcLock(this.root, `head-${lease.guid}`, async () => {
|
|
473
|
+
const current = await this.read(lease.guid);
|
|
474
|
+
if (current?.active?.attempt !== lease.attempt) {
|
|
475
|
+
throw new DdcStoreError({
|
|
476
|
+
code: 'ddc-lease-expired',
|
|
477
|
+
detail: 'heartbeat belongs to a stale lease',
|
|
478
|
+
expected: lease.attempt,
|
|
479
|
+
actual: current?.active?.attempt,
|
|
480
|
+
lease: lease.attempt,
|
|
481
|
+
generation: lease.generation,
|
|
482
|
+
rootKind: 'project-ddc',
|
|
483
|
+
});
|
|
484
|
+
}
|
|
485
|
+
const refreshed = { ...lease, expiresAt: Date.now() + this.leaseTtlMs };
|
|
486
|
+
await this.write({ ...current, active: refreshed });
|
|
487
|
+
return refreshed;
|
|
488
|
+
});
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
public async close(lease: DdcLease): Promise<void> {
|
|
492
|
+
await withDdcLock(this.root, `head-${lease.guid}`, async () => {
|
|
493
|
+
const current = await this.read(lease.guid);
|
|
494
|
+
if (current?.active?.instanceId !== lease.instanceId) return;
|
|
495
|
+
const { active: _active, ...withoutActive } = current;
|
|
496
|
+
await this.write({
|
|
497
|
+
...withoutActive,
|
|
498
|
+
revision: current.revision + 1,
|
|
499
|
+
stale: current.currentKey === undefined,
|
|
500
|
+
});
|
|
501
|
+
});
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
public async discard(lease: DdcLease): Promise<void> {
|
|
505
|
+
await withDdcLock(this.root, `head-${lease.guid}`, async () => {
|
|
506
|
+
const current = await this.read(lease.guid);
|
|
507
|
+
if (current?.active?.attempt !== lease.attempt) return;
|
|
508
|
+
const { active: _active, ...withoutActive } = current;
|
|
509
|
+
await this.write({
|
|
510
|
+
...withoutActive,
|
|
511
|
+
revision: current.revision + 1,
|
|
512
|
+
generation: lease.generation,
|
|
513
|
+
stale: current.currentKey === undefined,
|
|
514
|
+
});
|
|
515
|
+
});
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
/**
|
|
519
|
+
* Restore accepted content only when this lease still owns the mutable head.
|
|
520
|
+
* A newer active or terminal mutation makes rollback a deliberate no-op.
|
|
521
|
+
*/
|
|
522
|
+
public async restore(
|
|
523
|
+
head: DdcHead | DdcRollbackSnapshot,
|
|
524
|
+
lease?: DdcLease,
|
|
525
|
+
fence?: DdcRestoreFence,
|
|
526
|
+
): Promise<DdcRestoreResult> {
|
|
527
|
+
const inferredLease =
|
|
528
|
+
lease ??
|
|
529
|
+
('activeLease' in head && head.activeLease !== undefined ? head.activeLease : undefined);
|
|
530
|
+
if (inferredLease === undefined) {
|
|
531
|
+
return {
|
|
532
|
+
result: 'not-owner',
|
|
533
|
+
revision: head.revision ?? 0,
|
|
534
|
+
...(head.generation === undefined ? {} : { generation: head.generation }),
|
|
535
|
+
};
|
|
536
|
+
}
|
|
537
|
+
const snapshot = this.rollbackSnapshot(head);
|
|
538
|
+
return this.restoreIfCurrent(snapshot, inferredLease, fence);
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
public async restoreIfCurrent(
|
|
542
|
+
snapshot: DdcRollbackSnapshot,
|
|
543
|
+
lease: DdcLease,
|
|
544
|
+
fence?: DdcRestoreFence,
|
|
545
|
+
): Promise<DdcRestoreResult> {
|
|
546
|
+
return withDdcLock(this.root, `head-${lease.guid}`, async () => {
|
|
547
|
+
const current = await this.read(lease.guid);
|
|
548
|
+
if (current === null || current.guid !== snapshot.guid) {
|
|
549
|
+
return {
|
|
550
|
+
result: 'not-owner',
|
|
551
|
+
revision: current?.revision ?? 0,
|
|
552
|
+
...(current?.generation === undefined ? {} : { generation: current.generation }),
|
|
553
|
+
};
|
|
554
|
+
}
|
|
555
|
+
const ownsActive =
|
|
556
|
+
current.active?.attempt === lease.attempt &&
|
|
557
|
+
current.active.generation === lease.generation &&
|
|
558
|
+
current.active.desiredKey === lease.desiredKey;
|
|
559
|
+
const ownsTerminal = fence !== undefined && this.matchesRestoreFence(current, lease, fence);
|
|
560
|
+
if (!ownsActive && !ownsTerminal) {
|
|
561
|
+
return {
|
|
562
|
+
result: 'not-owner',
|
|
563
|
+
revision: current.revision,
|
|
564
|
+
...(current.generation === undefined ? {} : { generation: current.generation }),
|
|
565
|
+
};
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
const nextRevision = current.revision + 1;
|
|
569
|
+
const generation = Math.max(
|
|
570
|
+
current.generation ?? 0,
|
|
571
|
+
lease.generation,
|
|
572
|
+
snapshot.generation ?? 0,
|
|
573
|
+
);
|
|
574
|
+
const supersededAttempts = current.supersededAttempts;
|
|
575
|
+
const restored: HeadRecord = {
|
|
576
|
+
guid: snapshot.guid,
|
|
577
|
+
desiredKey: snapshot.desiredKey,
|
|
578
|
+
revision: nextRevision,
|
|
579
|
+
generation,
|
|
580
|
+
...(snapshot.currentKey === undefined ? {} : { currentKey: snapshot.currentKey }),
|
|
581
|
+
...(snapshot.lastKnownGoodKey === undefined
|
|
582
|
+
? {}
|
|
583
|
+
: { lastKnownGoodKey: snapshot.lastKnownGoodKey }),
|
|
584
|
+
...(supersededAttempts === undefined ? {} : { supersededAttempts }),
|
|
585
|
+
...(snapshot.state === 'missing' ? { empty: true } : {}),
|
|
586
|
+
...(snapshot.state === 'stale' ? { stale: true } : {}),
|
|
587
|
+
...(snapshot.failure === undefined
|
|
588
|
+
? {}
|
|
589
|
+
: { failure: { desiredKey: snapshot.desiredKey, ...snapshot.failure } }),
|
|
590
|
+
};
|
|
591
|
+
await this.write(restored);
|
|
592
|
+
return { result: 'restored', revision: nextRevision, generation };
|
|
593
|
+
});
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
public async revoke(lease: DdcLease): Promise<void> {
|
|
597
|
+
await this.fail(lease, {
|
|
598
|
+
code: 'lease-lost',
|
|
599
|
+
detail: 'cook lease was revoked before validation',
|
|
600
|
+
});
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
public async recover(guid: string, desiredKey: string): Promise<DdcHead> {
|
|
604
|
+
return withDdcLock(this.root, `head-${guid}`, async () => {
|
|
605
|
+
const current = await this.read(guid);
|
|
606
|
+
if (current?.active?.desiredKey === desiredKey) {
|
|
607
|
+
await this.write({
|
|
608
|
+
guid,
|
|
609
|
+
desiredKey,
|
|
610
|
+
revision: current.revision + 1,
|
|
611
|
+
...(current.generation === undefined ? {} : { generation: current.generation }),
|
|
612
|
+
...(current.supersededAttempts === undefined
|
|
613
|
+
? {}
|
|
614
|
+
: { supersededAttempts: current.supersededAttempts }),
|
|
615
|
+
...(current.lastKnownGoodKey === undefined
|
|
616
|
+
? {}
|
|
617
|
+
: { lastKnownGoodKey: current.lastKnownGoodKey }),
|
|
618
|
+
failure: {
|
|
619
|
+
desiredKey,
|
|
620
|
+
code: 'writer-crashed',
|
|
621
|
+
detail: 'active cook attempt was recovered after process interruption',
|
|
622
|
+
},
|
|
623
|
+
});
|
|
624
|
+
}
|
|
625
|
+
return this.inspect(guid, desiredKey);
|
|
626
|
+
});
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
private rollbackSnapshot(head: DdcHead | DdcRollbackSnapshot): DdcRollbackSnapshot {
|
|
630
|
+
const { activeLease: _activeLease, ...snapshot } = head as DdcHead;
|
|
631
|
+
if (snapshot.state !== 'cooking') return snapshot;
|
|
632
|
+
return {
|
|
633
|
+
...snapshot,
|
|
634
|
+
state: snapshot.currentKey === undefined ? 'missing' : 'stale',
|
|
635
|
+
};
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
private matchesRestoreFence(
|
|
639
|
+
current: HeadRecord,
|
|
640
|
+
lease: DdcLease,
|
|
641
|
+
fence: DdcRestoreFence,
|
|
642
|
+
): boolean {
|
|
643
|
+
if (
|
|
644
|
+
fence.attempt !== lease.attempt ||
|
|
645
|
+
fence.generation !== lease.generation ||
|
|
646
|
+
current.active !== undefined ||
|
|
647
|
+
current.revision !== fence.revision ||
|
|
648
|
+
current.generation !== fence.generation ||
|
|
649
|
+
current.desiredKey !== fence.desiredKey ||
|
|
650
|
+
current.currentKey !== fence.currentKey ||
|
|
651
|
+
current.lastKnownGoodKey !== fence.lastKnownGoodKey
|
|
652
|
+
) {
|
|
653
|
+
return false;
|
|
654
|
+
}
|
|
655
|
+
if (fence.outcome === 'current') {
|
|
656
|
+
return (
|
|
657
|
+
fence.key !== undefined &&
|
|
658
|
+
current.currentKey === fence.key &&
|
|
659
|
+
current.failure === undefined &&
|
|
660
|
+
current.stale !== true
|
|
661
|
+
);
|
|
662
|
+
}
|
|
663
|
+
if (
|
|
664
|
+
fence.failure === undefined ||
|
|
665
|
+
current.failure === undefined ||
|
|
666
|
+
current.failure.desiredKey !== fence.desiredKey ||
|
|
667
|
+
current.failure.code !== fence.failure.code ||
|
|
668
|
+
current.failure.detail !== fence.failure.detail
|
|
669
|
+
) {
|
|
670
|
+
return false;
|
|
671
|
+
}
|
|
672
|
+
return fence.outcome === 'invalid' ? fence.currentKey === undefined : true;
|
|
673
|
+
}
|
|
674
|
+
|
|
675
|
+
private async projectHead(
|
|
676
|
+
guid: string,
|
|
677
|
+
desiredKey: string,
|
|
678
|
+
record: HeadRecord | null,
|
|
679
|
+
includeActive = true,
|
|
680
|
+
): Promise<DdcHead> {
|
|
681
|
+
if (record === null) {
|
|
682
|
+
return {
|
|
683
|
+
guid,
|
|
684
|
+
desiredKey,
|
|
685
|
+
state: 'missing',
|
|
686
|
+
currentKey: undefined,
|
|
687
|
+
lastKnownGoodKey: undefined,
|
|
688
|
+
revision: 0,
|
|
689
|
+
};
|
|
690
|
+
}
|
|
691
|
+
const recordedFailure =
|
|
692
|
+
record.failure?.desiredKey === desiredKey
|
|
693
|
+
? { code: record.failure.code, detail: record.failure.detail }
|
|
694
|
+
: undefined;
|
|
695
|
+
const currentEntry =
|
|
696
|
+
record.currentKey === undefined ? null : await this.entries.readChecked(record.currentKey);
|
|
697
|
+
const entryFailure =
|
|
698
|
+
currentEntry !== null && !currentEntry.ok
|
|
699
|
+
? { code: currentEntry.error.code, detail: currentEntry.error.detail }
|
|
700
|
+
: undefined;
|
|
701
|
+
const failure = recordedFailure ?? entryFailure;
|
|
702
|
+
const currentValue = currentEntry?.ok === true ? currentEntry.value : null;
|
|
703
|
+
const state: DdcLifecycleState =
|
|
704
|
+
failure !== undefined
|
|
705
|
+
? 'failed'
|
|
706
|
+
: record.currentKey === desiredKey && currentValue?.guid === guid
|
|
707
|
+
? 'current'
|
|
708
|
+
: record.stale === true
|
|
709
|
+
? 'stale'
|
|
710
|
+
: includeActive && record.active?.desiredKey === desiredKey
|
|
711
|
+
? 'cooking'
|
|
712
|
+
: record.empty === true
|
|
713
|
+
? 'missing'
|
|
714
|
+
: 'stale';
|
|
715
|
+
return {
|
|
716
|
+
guid,
|
|
717
|
+
desiredKey,
|
|
718
|
+
state,
|
|
719
|
+
currentKey: record.currentKey,
|
|
720
|
+
lastKnownGoodKey: record.lastKnownGoodKey,
|
|
721
|
+
revision: record.revision,
|
|
722
|
+
...(record.generation === undefined ? {} : { generation: record.generation }),
|
|
723
|
+
...(includeActive && record.active === undefined
|
|
724
|
+
? {}
|
|
725
|
+
: includeActive && record.active !== undefined
|
|
726
|
+
? { activeLease: record.active }
|
|
727
|
+
: {}),
|
|
728
|
+
...(failure === undefined ? {} : { failure }),
|
|
729
|
+
};
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
private async allocateGeneration(minimumExclusive = 0): Promise<number> {
|
|
733
|
+
return withDdcLock(this.root, 'generation-counter', async () => {
|
|
734
|
+
const path = join(this.root, 'generations', 'counter.json');
|
|
735
|
+
await mkdir(join(this.root, 'generations'), { recursive: true });
|
|
736
|
+
let next = 1;
|
|
737
|
+
try {
|
|
738
|
+
const record = JSON.parse(await readFile(path, 'utf8')) as GenerationRecord;
|
|
739
|
+
if (
|
|
740
|
+
record.schemaVersion === 'forgeax-ddc-generation/v2' &&
|
|
741
|
+
Number.isSafeInteger(record.next) &&
|
|
742
|
+
record.next >= 1
|
|
743
|
+
)
|
|
744
|
+
next = Math.max(1, record.next);
|
|
745
|
+
} catch {
|
|
746
|
+
// A missing counter is the first allocation, not an implicit legacy fallback.
|
|
747
|
+
}
|
|
748
|
+
const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`;
|
|
749
|
+
next = Math.max(next, minimumExclusive + 1);
|
|
750
|
+
await writeFile(
|
|
751
|
+
temporary,
|
|
752
|
+
JSON.stringify({ schemaVersion: 'forgeax-ddc-generation/v2', next: next + 1 }),
|
|
753
|
+
);
|
|
754
|
+
await rename(temporary, path);
|
|
755
|
+
return next;
|
|
756
|
+
});
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
private async read(guid: string): Promise<HeadRecord | null> {
|
|
760
|
+
const path = headFile(this.heads, guid);
|
|
761
|
+
let source: string;
|
|
762
|
+
try {
|
|
763
|
+
source = await readFile(path, 'utf8');
|
|
764
|
+
} catch (error) {
|
|
765
|
+
if (
|
|
766
|
+
error !== null &&
|
|
767
|
+
typeof error === 'object' &&
|
|
768
|
+
'code' in error &&
|
|
769
|
+
(error as { readonly code?: unknown }).code === 'ENOENT'
|
|
770
|
+
) {
|
|
771
|
+
return null;
|
|
772
|
+
}
|
|
773
|
+
throw malformedHeadError('unreadable');
|
|
774
|
+
}
|
|
775
|
+
let value: unknown;
|
|
776
|
+
try {
|
|
777
|
+
value = JSON.parse(source);
|
|
778
|
+
} catch {
|
|
779
|
+
throw malformedHeadError('syntax-invalid');
|
|
780
|
+
}
|
|
781
|
+
if (!isHeadRecord(value) || value.guid !== guid) throw malformedHeadError('schema-invalid');
|
|
782
|
+
return value;
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
private async write(record: HeadRecord): Promise<void> {
|
|
786
|
+
await mkdir(this.heads, { recursive: true });
|
|
787
|
+
const path = headFile(this.heads, record.guid);
|
|
788
|
+
const temporary = `${path}.${randomUUID()}.tmp`;
|
|
789
|
+
await writeFile(temporary, JSON.stringify(record));
|
|
790
|
+
await rename(temporary, path);
|
|
791
|
+
}
|
|
792
|
+
}
|